sascha/vlfi
1
0
Fork 0
mirror of https://github.com/m00natic/vlfi.git synced 2025-02-07 05:30:47 +00:00
Code Issues Releases Wiki Activity

Add linear search for tuning and prefer smaller batches.

Browse Source
This commit is contained in:
Andrey Kotlarski 2014-09-05 02:49:55 +03:00
parent d85f3d43fc
commit e18a05b7cb
2 changed files with 106 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
vlf-occur.el
View File

@ -200,10 +200,8 @@ Prematurely ending indexing will still show what's found so far."
(is-hexl (derived-mode-p 'hexl-mode)) (is-hexl (derived-mode-p 'hexl-mode))
(end-of-file nil) (end-of-file nil)
(time (float-time)) (time (float-time))
(tune-types (let ((base '(:insert :encode))) (tune-types (if is-hexl '(:hexl :dehexlify :insert :encode)
(if is-hexl '(:insert :encode)))
(append '(:hexl :dehexlify) base)
base)))
(reporter (make-progress-reporter (reporter (make-progress-reporter
(concat "Building index for " regexp "...") (concat "Building index for " regexp "...")
vlf-start-pos vlf-file-size))) vlf-start-pos vlf-file-size)))
@ -259,7 +257,7 @@ Prematurely ending indexing will still show what's found so far."
total-matches)))))))) total-matches))))))))
(setq end-of-file (= vlf-end-pos vlf-file-size)) (setq end-of-file (= vlf-end-pos vlf-file-size))
(unless end-of-file (unless end-of-file
(vlf-tune-best tune-types) (vlf-tune-optimal tune-types)
(let ((batch-move (- vlf-end-pos batch-step))) (let ((batch-move (- vlf-end-pos batch-step)))
(vlf-move-to-batch (if (or is-hexl (vlf-move-to-batch (if (or is-hexl
(< match-end-pos (< match-end-pos
@ -302,8 +300,8 @@ in file: %s" total-matches line regexp file)
(message "Occur finished for \"%s\" (%f secs)" (message "Occur finished for \"%s\" (%f secs)"
regexp (- (float-time) time)))))) regexp (- (float-time) time))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; save, load vlf-occur data ;;; save, load vlf-occur data
(defun vlf-occur-save (file) (defun vlf-occur-save (file)
"Serialize `vlf-occur' results to FILE which can later be reloaded." "Serialize `vlf-occur' results to FILE which can later be reloaded."

174
vlf-tune.el
View File

@ -36,17 +36,18 @@ but don't change batch size. If t, measure and change."
(const :tag "Just statistics" stats) (const :tag "Just statistics" stats)
(const :tag "Disabled" nil))) (const :tag "Disabled" nil)))
(defvar vlf-file-size 0 "Total size of presented file.") (defvar vlf-file-size 0 "Total size in bytes of presented file.")
(make-variable-buffer-local 'vlf-file-size) (make-variable-buffer-local 'vlf-file-size)
(put 'vlf-file-size 'permanent-local t) (put 'vlf-file-size 'permanent-local t)
(defun vlf-tune-ram-size () (defun vlf-tune-ram-size ()
"Try to determine RAM size in bytes." "Try to determine RAM size in bytes."
(let* ((free-output (shell-command-to-string "free")) (if (executable-find "free")
(match-from (string-match "[[:digit:]]+" free-output))) (let* ((free (shell-command-to-string "free"))
(if match-from (match-from (string-match "[[:digit:]]+" free)))
(* 1000 (string-to-number (substring free-output match-from (if match-from
(match-end 0))))))) (* 1000 (string-to-number (substring free match-from
(match-end 0))))))))
(defcustom vlf-tune-max (let ((ram-size (vlf-tune-ram-size))) (defcustom vlf-tune-max (let ((ram-size (vlf-tune-ram-size)))
(if ram-size (if ram-size
@ -89,15 +90,18 @@ but don't change batch size. If t, measure and change."
(make-variable-buffer-local 'vlf-tune-dehexlify-bps) (make-variable-buffer-local 'vlf-tune-dehexlify-bps)
(put 'vlf-tune-dehexlify-bps 'permanent-local t) (put 'vlf-tune-dehexlify-bps 'permanent-local t)
(defun vlf-tune-initialize-measurement ()
"Initialize measurement vector."
(make-vector (1- (/ vlf-tune-max vlf-tune-step)) '(0 . 0)))
(defun vlf-tune-closest-index (size) (defun vlf-tune-closest-index (size)
"Get closest measurement index corresponding to SIZE." "Get closest measurement index corresponding to SIZE."
(let ((step (float vlf-tune-step))) (let ((step (float vlf-tune-step)))
(max 0 (1- (min (round size step) (round vlf-tune-max step)))))) (max 0 (1- (min (round size step) (round vlf-tune-max step))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; bookkeeping
(defun vlf-tune-initialize-measurement ()
"Initialize measurement vector."
(make-vector (1- (/ vlf-tune-max vlf-tune-step)) '(0 . 0)))
(defmacro vlf-tune-add-measurement (vec size time) (defmacro vlf-tune-add-measurement (vec size time)
"Add at an appropriate position in VEC new SIZE TIME measurement. "Add at an appropriate position in VEC new SIZE TIME measurement.
VEC is a vector of (mean time . count) elements ordered by size." VEC is a vector of (mean time . count) elements ordered by size."
@ -162,6 +166,9 @@ SIZE is number of bytes that are saved."
(vlf-tune-add-measurement vlf-tune-dehexlify-bps (vlf-tune-add-measurement vlf-tune-dehexlify-bps
hexl-max-address time))) hexl-max-address time)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; tuning
(defun vlf-tune-assess (type coef index) (defun vlf-tune-assess (type coef index)
"Get measurement value according to TYPE, COEF and INDEX." "Get measurement value according to TYPE, COEF and INDEX."
(* coef (or (cond ((eq type :insert) (* coef (or (cond ((eq type :insert)
@ -194,84 +201,105 @@ SIZE is number of bytes that are saved."
0))) 0)))
(defun vlf-tune-score (types index) (defun vlf-tune-score (types index)
"Cumulative speed over TYPES which is alist of (type coef) for INDEX." "Calculate cumulative speed over TYPES for INDEX."
(catch 'result (catch 'result
(let ((score 0) (let ((time 0)
(size (* (1+ index) vlf-tune-step))) (size (* (1+ index) vlf-tune-step)))
(dolist (el types (/ size score)) (dolist (el types (/ size time))
(let ((bps (if (consp el) (let ((bps (if (consp el)
(vlf-tune-assess (car el) (cadr el) index) (vlf-tune-assess (car el) (cadr el) index)
(vlf-tune-assess el 1 index)))) (vlf-tune-assess el 1 index))))
(if (zerop bps) (if (zerop bps)
(throw 'result nil) (throw 'result nil)
(setq score (+ score (/ size bps))))))))) (setq time (+ time (/ size bps)))))))))
(defun vlf-tune-conservative (types &optional index) (defun vlf-tune-conservative (types &optional index)
"Adjust `vlf-batch-size' with `vlf-tune-step' in case of better score. "Adjust `vlf-batch-size' to best nearby value over TYPES.
Score is calculated over TYPES which is alist of form (type coef).
INDEX if given, specifies search independent of current batch size." INDEX if given, specifies search independent of current batch size."
(if (eq vlf-tune-enabled t) (if (eq vlf-tune-enabled t)
(let* ((half-max (/ vlf-file-size 2)) (let* ((half-max (/ vlf-file-size 2))
(idx (or index (vlf-tune-closest-index vlf-batch-size))) (idx (or index (vlf-tune-closest-index vlf-batch-size)))
(curr (if (< half-max (* idx vlf-tune-step)) (curr (if (< half-max (* idx vlf-tune-step)) t
t
(vlf-tune-score types idx)))) (vlf-tune-score types idx))))
(if (null curr) (if curr
(setq vlf-batch-size (* (1+ idx) vlf-tune-step)) (let ((prev (if (zerop idx) t
(let ((next (if (or (eq curr t) (vlf-tune-score types (1- idx)))))
(< half-max (* (1+ idx) vlf-tune-step))) (if prev
t (let ((next (if (or (eq curr t)
(vlf-tune-score types (1+ idx))))) (< half-max (* (1+ idx)
(if (null next) vlf-tune-step)))
(setq vlf-batch-size (* (+ idx 2) vlf-tune-step)) t
(let ((prev (if (zerop idx) (vlf-tune-score types (1+ idx)))))
t (cond ((null next)
(vlf-tune-score types (1- idx))))) (setq vlf-batch-size (* (+ 2 idx)
(cond ((null prev)
(setq vlf-batch-size (* idx vlf-tune-step)))
((eq curr t)
(or (eq prev t)
(setq vlf-batch-size (* idx
vlf-tune-step))))
(t (let ((best-idx idx))
(and (numberp next) (< curr next)
(setq curr next
best-idx (1+ idx)))
(and (numberp prev) (< curr prev)
(setq best-idx (1- idx)))
(setq vlf-batch-size
(* (1+ best-idx)
vlf-tune-step))))))))))))
(defun vlf-tune-best (types &optional min max)
"Adjust `vlf-batch-size' to optional value with binary search.
Score is calculated over TYPES which is alist of form (type coef).
MIN and MAX may specify interval of indexes to search."
(if (eq vlf-tune-enabled t)
(if (and (null min) (file-remote-p buffer-file-name))
(vlf-tune-conservative types)
(setq min (or min 0)
max (or max (1- (/ (min vlf-tune-max
(/ vlf-file-size 2))
vlf-tune-step))))
(or (< max 1)
(if (< (- max min) 3)
(vlf-tune-conservative types (round (+ min max) 2))
(let* ((right-idx (round (+ min (* 3 max)) 4))
(right (vlf-tune-score types right-idx)))
(if (null right)
(setq vlf-batch-size (* (1+ right-idx)
vlf-tune-step))
(let* ((left-idx (round (+ (* 3 min) max) 4))
(left (vlf-tune-score types left-idx)))
(cond ((null left)
(setq vlf-batch-size (* (1+ left-idx)
vlf-tune-step))) vlf-tune-step)))
((< right left) ((eq curr t)
(vlf-tune-best types min (or (eq prev t)
(round (+ max min) 2))) (setq vlf-batch-size
(t (vlf-tune-best types (round (+ max min) 2) (* idx vlf-tune-step))))
max))))))))))) (t (let ((best-idx idx))
(and (numberp prev) (< curr prev)
(setq curr prev
best-idx (1- idx)))
(and (numberp next) (< curr next)
(setq best-idx (1+ idx)))
(setq vlf-batch-size
(* (1+ best-idx)
vlf-tune-step))))))
(setq vlf-batch-size (* idx vlf-tune-step))))
(setq vlf-batch-size (* (1+ idx) vlf-tune-step))))))
(defun vlf-tune-binary (types min max)
"Adjust `vlf-batch-size' to optimal value using binary search, \
optimizing over TYPES.
MIN and MAX specify interval of indexes to search."
(let* ((left-idx (round (+ (* 3 min) max) 4))
(left (vlf-tune-score types left-idx)))
(if left
(let* ((right-idx (round (+ min (* 3 max)) 4))
(right (vlf-tune-score types right-idx)))
(cond ((null right)
(setq vlf-batch-size (* (1+ right-idx)
vlf-tune-step)))
((< left right)
(vlf-tune-binary types (/ (+ 1 max min) 2) max))
(t (vlf-tune-binary types min (/ (+ max min) 2)))))
(setq vlf-batch-size (* (1+ left-idx) vlf-tune-step)))))
(defun vlf-tune-linear (types max-idx)
"Adjust `vlf-batch-size' to optimal value using linear search, \
optimizing over TYPES up to MAX-IDX."
(let ((best-idx 0)
(best-bps 0)
(idx 0)
(none-missing t))
(while (and none-missing (<= idx max-idx))
(let ((bps (vlf-tune-score types idx)))
(cond ((null bps)
(setq vlf-batch-size (* (1+ idx) vlf-tune-step)
none-missing nil))
((< best-bps bps) (setq best-idx idx
best-bps bps))))
(setq idx (1+ idx)))
(or (not none-missing)
(setq vlf-batch-size (* (1+ best-idx) vlf-tune-step)))))
(defun vlf-tune-optimal (types &optional linear)
"Adjust `vlf-batch-size' to optimal value optimizing on TYPES.
TYPES is alist of elements that may be of form (type coef) or
non list values in which case coeficient is assumed 1.
Types can be :insert, :raw, :encode, :write, :hexl or :dehexlify.
If LINEAR is non nil, use brute-force."
(if (eq vlf-tune-enabled t)
(let ((max-idx (1- (/ (min vlf-tune-max (/ vlf-file-size 2))
vlf-tune-step))))
(cond (linear (vlf-tune-linear types max-idx))
((file-remote-p buffer-file-name)
(vlf-tune-conservative types))
((<= 1 max-idx)
(if (< max-idx 3)
(vlf-tune-conservative types (/ max-idx 2))
(vlf-tune-binary types 0 max-idx)))))))
(provide 'vlf-tune) (provide 'vlf-tune)