sascha/vlfi
1
0
Fork 0
mirror of https://github.com/m00natic/vlfi.git synced 2025-01-31 02:00:47 +00:00
Code Issues Releases Wiki Activity

Apply batch size tuning on adjacent moves, search, save adjusting and

Browse Source 
report total times.
This commit is contained in:
Andrey Kotlarski 2014-09-05 02:52:32 +03:00
parent e18a05b7cb
commit e8bb4a91da
3 changed files with 70 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
vlf-search.el
View File

@ -29,12 +29,21 @@
(require 'vlf) (require 'vlf)
(defun vlf-re-search (regexp count backward batch-step) (defun vlf-re-search (regexp count backward batch-step
&optional reporter time)
"Search for REGEXP COUNT number of times forward or BACKWARD. "Search for REGEXP COUNT number of times forward or BACKWARD.
BATCH-STEP is amount of overlap between successive chunks." BATCH-STEP is amount of overlap between successive chunks.
Use existing REPORTER and start TIME if given."
(if (<= count 0) (if (<= count 0)
(error "Count must be positive")) (error "Count must be positive"))
(run-hook-with-args 'vlf-before-batch-functions 'search) (run-hook-with-args 'vlf-before-batch-functions 'search)
(or reporter (setq reporter (make-progress-reporter
(concat "Searching for " regexp "...")
(if backward
(- vlf-file-size vlf-end-pos)
vlf-start-pos)
vlf-file-size)))
(or time (setq time (float-time)))
(let* ((tramp-verbose (if (boundp 'tramp-verbose) (let* ((tramp-verbose (if (boundp 'tramp-verbose)
(min tramp-verbose 2))) (min tramp-verbose 2)))
(case-fold-search t) (case-fold-search t)
@ -44,13 +53,9 @@ BATCH-STEP is amount of overlap between successive chunks."
(match-end-pos match-start-pos) (match-end-pos match-start-pos)
(to-find count) (to-find count)
(is-hexl (derived-mode-p 'hexl-mode)) (is-hexl (derived-mode-p 'hexl-mode))
(font-lock font-lock-mode) (tune-types (if is-hexl '(:hexl :dehexlify :insert :encode)
(reporter (make-progress-reporter '(:insert :encode)))
(concat "Searching for " regexp "...") (font-lock font-lock-mode))
(if backward
(- vlf-file-size vlf-end-pos)
vlf-start-pos)
vlf-file-size)))
(font-lock-mode 0) (font-lock-mode 0)
(vlf-with-undo-disabled (vlf-with-undo-disabled
(unwind-protect (unwind-protect
@ -69,7 +74,8 @@ BATCH-STEP is amount of overlap between successive chunks."
(match-end 0))))) (match-end 0)))))
((zerop vlf-start-pos) ((zerop vlf-start-pos)
(throw 'end-of-file nil)) (throw 'end-of-file nil))
(t (let ((batch-move (- vlf-start-pos (t (vlf-tune-optimal tune-types)
(let ((batch-move (- vlf-start-pos
(- vlf-batch-size (- vlf-batch-size
batch-step)))) batch-step))))
(vlf-move-to-batch (vlf-move-to-batch
@ -82,9 +88,9 @@ BATCH-STEP is amount of overlap between successive chunks."
match-start-pos)) match-start-pos))
(point-max) (point-max)
(or (byte-to-position (or (byte-to-position
(- match-start-pos (- match-start-pos
vlf-start-pos)) vlf-start-pos))
(point-max)))) (point-max))))
(progress-reporter-update (progress-reporter-update
reporter (- vlf-file-size reporter (- vlf-file-size
vlf-start-pos))))) vlf-start-pos)))))
@ -101,7 +107,8 @@ BATCH-STEP is amount of overlap between successive chunks."
(match-end 0))))) (match-end 0)))))
((= vlf-end-pos vlf-file-size) ((= vlf-end-pos vlf-file-size)
(throw 'end-of-file nil)) (throw 'end-of-file nil))
(t (let ((batch-move (- vlf-end-pos batch-step))) (t (vlf-tune-optimal tune-types)
(let ((batch-move (- vlf-end-pos batch-step)))
(vlf-move-to-batch (vlf-move-to-batch
(if (or is-hexl (if (or is-hexl
(< match-end-pos batch-move)) (< match-end-pos batch-move))
@ -111,9 +118,9 @@ BATCH-STEP is amount of overlap between successive chunks."
(<= match-end-pos vlf-start-pos)) (<= match-end-pos vlf-start-pos))
(point-min) (point-min)
(or (byte-to-position (or (byte-to-position
(- match-end-pos (- match-end-pos
vlf-start-pos)) vlf-start-pos))
(point-min)))) (point-min))))
(progress-reporter-update reporter (progress-reporter-update reporter
vlf-end-pos))))) vlf-end-pos)))))
(progress-reporter-done reporter)) (progress-reporter-done reporter))
@ -122,26 +129,26 @@ BATCH-STEP is amount of overlap between successive chunks."
(if backward (if backward
(vlf-goto-match match-chunk-start match-chunk-end (vlf-goto-match match-chunk-start match-chunk-end
match-end-pos match-start-pos match-end-pos match-start-pos
count to-find) count to-find time)
(vlf-goto-match match-chunk-start match-chunk-end (vlf-goto-match match-chunk-start match-chunk-end
match-start-pos match-end-pos match-start-pos match-end-pos
count to-find)) count to-find time))
(run-hook-with-args 'vlf-after-batch-functions 'search))))) (run-hook-with-args 'vlf-after-batch-functions 'search)))))
(defun vlf-goto-match (match-chunk-start match-chunk-end (defun vlf-goto-match (match-chunk-start match-chunk-end
match-pos-start match-pos-start match-pos-end
match-pos-end count to-find time)
count to-find)
"Move to MATCH-CHUNK-START MATCH-CHUNK-END surrounding\ "Move to MATCH-CHUNK-START MATCH-CHUNK-END surrounding\
MATCH-POS-START and MATCH-POS-END. MATCH-POS-START and MATCH-POS-END.
According to COUNT and left TO-FIND, show if search has been According to COUNT and left TO-FIND, show if search has been
successful. Return nil if nothing found." successful. Use start TIME to report how much it took.
Return nil if nothing found."
(if (= count to-find) (if (= count to-find)
(progn (vlf-move-to-chunk match-chunk-start match-chunk-end) (progn (vlf-move-to-chunk match-chunk-start match-chunk-end)
(goto-char (or (byte-to-position (- match-pos-start (goto-char (or (byte-to-position (- match-pos-start
vlf-start-pos)) vlf-start-pos))
(point-max))) (point-max)))
(message "Not found") (message "Not found (%f secs)" (- (float-time) time))
nil) nil)
(let ((success (zerop to-find))) (let ((success (zerop to-find)))
(if success (if success
@ -155,10 +162,11 @@ successful. Return nil if nothing found."
vlf-start-pos)) vlf-start-pos))
match-end))) match-end)))
(overlay-put overlay 'face 'match) (overlay-put overlay 'face 'match)
(unless success (if success
(message "Match found (%f secs)" (- (float-time) time))
(goto-char match-end) (goto-char match-end)
(message "Moved to the %d match which is last" (message "Moved to the %d match which is last (%f secs)"
(- count to-find))) (- count to-find) (- (float-time) time)))
(unwind-protect (sit-for 3) (unwind-protect (sit-for 3)
(delete-overlay overlay)) (delete-overlay overlay))
t)))) t))))
@ -171,7 +179,7 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(if regexp-history (if regexp-history
(car regexp-history))) (car regexp-history)))
(or current-prefix-arg 1)))) (or current-prefix-arg 1))))
(vlf-re-search regexp count nil (/ vlf-batch-size 8))) (vlf-re-search regexp count nil (min 1024 (/ vlf-batch-size 8))))
(defun vlf-re-search-backward (regexp count) (defun vlf-re-search-backward (regexp count)
"Search backward for REGEXP prefix COUNT number of times. "Search backward for REGEXP prefix COUNT number of times.
@ -181,7 +189,7 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(if regexp-history (if regexp-history
(car regexp-history))) (car regexp-history)))
(or current-prefix-arg 1)))) (or current-prefix-arg 1))))
(vlf-re-search regexp count t (/ vlf-batch-size 8))) (vlf-re-search regexp count t (min 1024 (/ vlf-batch-size 8))))
(defun vlf-goto-line (n) (defun vlf-goto-line (n)
"Go to line N. If N is negative, count from the end of file." "Go to line N. If N is negative, count from the end of file."
@ -196,8 +204,10 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(pos (point)) (pos (point))
(is-hexl (derived-mode-p 'hexl-mode)) (is-hexl (derived-mode-p 'hexl-mode))
(font-lock font-lock-mode) (font-lock font-lock-mode)
(time (float-time))
(success nil)) (success nil))
(font-lock-mode 0) (font-lock-mode 0)
(vlf-tune-optimal '(:raw))
(unwind-protect (unwind-protect
(if (< 0 n) (if (< 0 n)
(let ((start 0) (let ((start 0)
@ -218,6 +228,7 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(while (re-search-forward "[\n\C-m]" nil t) (while (re-search-forward "[\n\C-m]" nil t)
(setq n (1- n))) (setq n (1- n)))
(vlf-verify-size) (vlf-verify-size)
(vlf-tune-optimal '(:raw))
(setq start end (setq start end
end (min vlf-file-size end (min vlf-file-size
(+ start vlf-batch-size))) (+ start vlf-batch-size)))
@ -225,7 +236,8 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(when (< n (- vlf-file-size end)) (when (< n (- vlf-file-size end))
(vlf-move-to-chunk-2 start end) (vlf-move-to-chunk-2 start end)
(goto-char (point-min)) (goto-char (point-min))
(setq success (vlf-re-search "[\n\C-m]" n nil 0))))) (setq success (vlf-re-search "[\n\C-m]" n nil 0
reporter time)))))
(let ((start (max 0 (- vlf-file-size vlf-batch-size))) (let ((start (max 0 (- vlf-file-size vlf-batch-size)))
(end vlf-file-size) (end vlf-file-size)
(reporter (make-progress-reporter (reporter (make-progress-reporter
@ -242,6 +254,7 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(goto-char (point-max)) (goto-char (point-max))
(while (re-search-backward "[\n\C-m]" nil t) (while (re-search-backward "[\n\C-m]" nil t)
(setq n (1- n))) (setq n (1- n)))
(vlf-tune-optimal '(:raw))
(setq end start (setq end start
start (max 0 (- end vlf-batch-size))) start (max 0 (- end vlf-batch-size)))
(progress-reporter-update reporter (progress-reporter-update reporter
@ -249,7 +262,8 @@ Search is performed chunk by chunk in `vlf-batch-size' memory."
(when (< n end) (when (< n end)
(vlf-move-to-chunk-2 start end) (vlf-move-to-chunk-2 start end)
(goto-char (point-max)) (goto-char (point-max))
(setq success (vlf-re-search "[\n\C-m]" n t 0)))))) (setq success (vlf-re-search "[\n\C-m]" n t 0
reporter time))))))
(if font-lock (font-lock-mode 1)) (if font-lock (font-lock-mode 1))
(unless success (unless success
(vlf-with-undo-disabled (vlf-with-undo-disabled

20
vlf-write.el
View File

@ -64,9 +64,11 @@ If changing size of chunk, shift remaining file content."
(pos (point)) (pos (point))
(font-lock font-lock-mode)) (font-lock font-lock-mode))
(font-lock-mode 0) (font-lock-mode 0)
(if (< 0 size-change) (let ((time (float-time)))
(vlf-file-shift-back size-change region-length) (if (< 0 size-change)
(vlf-file-shift-forward (- size-change) region-length)) (vlf-file-shift-back size-change region-length)
(vlf-file-shift-forward (- size-change) region-length))
(message "Save took %f seconds" (- (float-time) time)))
(if font-lock (font-lock-mode 1)) (if font-lock (font-lock-mode 1))
(vlf-move-to-chunk-2 vlf-start-pos (vlf-move-to-chunk-2 vlf-start-pos
(if (< (- vlf-end-pos vlf-start-pos) (if (< (- vlf-end-pos vlf-start-pos)
@ -76,7 +78,7 @@ If changing size of chunk, shift remaining file content."
(vlf-update-buffer-name) (vlf-update-buffer-name)
(goto-char pos))))) (goto-char pos)))))
(if hexl (vlf-tune-hexlify))) (if hexl (vlf-tune-hexlify)))
(run-hook-with-args 'vlf-after-batch-functions 'write)) (run-hook-with-args 'vlf-after-batch-functions 'write))
t) t)
(defun vlf-file-shift-back (size-change write-size) (defun vlf-file-shift-back (size-change write-size)
@ -106,6 +108,7 @@ WRITE-SIZE is byte length of saved chunk."
back at WRITE-POS. Return nil if EOF is reached, t otherwise." back at WRITE-POS. Return nil if EOF is reached, t otherwise."
(erase-buffer) (erase-buffer)
(vlf-verify-size t) (vlf-verify-size t)
(vlf-tune-optimal '(:raw :write))
(let ((read-end (min (+ read-pos vlf-batch-size) vlf-file-size))) (let ((read-end (min (+ read-pos vlf-batch-size) vlf-file-size)))
(vlf-tune-insert-file-contents-literally read-pos read-end) (vlf-tune-insert-file-contents-literally read-pos read-end)
(vlf-tune-write nil nil write-pos 0 (- read-end read-pos)) (vlf-tune-write nil nil write-pos 0 (- read-end read-pos))
@ -115,7 +118,8 @@ back at WRITE-POS. Return nil if EOF is reached, t otherwise."
"Shift file contents SIZE-CHANGE bytes forward. "Shift file contents SIZE-CHANGE bytes forward.
WRITE-SIZE is byte length of saved chunk. WRITE-SIZE is byte length of saved chunk.
Done by saving content up front and then writing previous batch." Done by saving content up front and then writing previous batch."
(let ((read-size (max (/ vlf-batch-size 2) size-change)) (vlf-tune-optimal '(:raw :write))
(let ((read-size (max vlf-batch-size size-change))
(read-pos vlf-end-pos) (read-pos vlf-end-pos)
(write-pos vlf-start-pos) (write-pos vlf-start-pos)
(reporter (make-progress-reporter "Adjusting file content..." (reporter (make-progress-reporter "Adjusting file content..."
@ -124,18 +128,20 @@ Done by saving content up front and then writing previous batch."
(vlf-with-undo-disabled (vlf-with-undo-disabled
(when (vlf-shift-batches read-size read-pos write-pos (when (vlf-shift-batches read-size read-pos write-pos
write-size t) write-size t)
(vlf-tune-optimal '(:raw :write))
(setq write-pos (+ read-pos size-change) (setq write-pos (+ read-pos size-change)
read-pos (+ read-pos read-size) read-pos (+ read-pos read-size)
write-size read-size write-size read-size
read-size (max (/ vlf-batch-size 2) size-change)) read-size (max vlf-batch-size size-change))
(progress-reporter-update reporter write-pos) (progress-reporter-update reporter write-pos)
(let ((coding-system-for-write 'no-conversion)) (let ((coding-system-for-write 'no-conversion))
(while (vlf-shift-batches read-size read-pos write-pos (while (vlf-shift-batches read-size read-pos write-pos
write-size nil) write-size nil)
(vlf-tune-optimal '(:raw :write))
(setq write-pos (+ read-pos size-change) (setq write-pos (+ read-pos size-change)
read-pos (+ read-pos read-size) read-pos (+ read-pos read-size)
write-size read-size write-size read-size
read-size (max (/ vlf-batch-size 2) size-change)) read-size (max vlf-batch-size size-change))
(progress-reporter-update reporter write-pos))))) (progress-reporter-update reporter write-pos)))))
(progress-reporter-done reporter))) (progress-reporter-done reporter)))

12
vlf.el
View File

@ -172,6 +172,9 @@ When prefix argument is negative
append next APPEND number of batches to the existing buffer." append next APPEND number of batches to the existing buffer."
(interactive "p") (interactive "p")
(vlf-verify-size) (vlf-verify-size)
(if (derived-mode-p 'hexl-mode)
(vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
(vlf-tune-conservative '(:insert :encode)))
(let* ((end (min (+ vlf-end-pos (* vlf-batch-size (abs append))) (let* ((end (min (+ vlf-end-pos (* vlf-batch-size (abs append)))
vlf-file-size)) vlf-file-size))
(start (if (< append 0) (start (if (< append 0)
@ -188,6 +191,9 @@ When prefix argument is negative
(interactive "p") (interactive "p")
(if (zerop vlf-start-pos) (if (zerop vlf-start-pos)
(error "Already at BOF")) (error "Already at BOF"))
(if (derived-mode-p 'hexl-mode)
(vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
(vlf-tune-conservative '(:insert :encode)))
(let* ((start (max 0 (- vlf-start-pos (* vlf-batch-size (abs prepend))))) (let* ((start (max 0 (- vlf-start-pos (* vlf-batch-size (abs prepend)))))
(end (if (< prepend 0) (end (if (< prepend 0)
vlf-end-pos vlf-end-pos
@ -260,12 +266,18 @@ with the prefix argument DECREASE it is halved."
(defun vlf-beginning-of-file () (defun vlf-beginning-of-file ()
"Jump to beginning of file content." "Jump to beginning of file content."
(interactive) (interactive)
(if (derived-mode-p 'hexl-mode)
(vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
(vlf-tune-conservative '(:insert :encode)))
(vlf-move-to-batch 0)) (vlf-move-to-batch 0))
(defun vlf-end-of-file () (defun vlf-end-of-file ()
"Jump to end of file content." "Jump to end of file content."
(interactive) (interactive)
(vlf-verify-size) (vlf-verify-size)
(if (derived-mode-p 'hexl-mode)
(vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
(vlf-tune-conservative '(:insert :encode)))
(vlf-move-to-batch vlf-file-size)) (vlf-move-to-batch vlf-file-size))
(defun vlf-revert (&optional _auto noconfirm) (defun vlf-revert (&optional _auto noconfirm)