Improve vlf performance with hexl-mode. Align batches to hexl-bits

width.

vlf-base.el

@@ -90,6 +90,9 @@ If same as current chunk is requested, do nothing.
 Return number of bytes moved back for proper decoding and number of
 bytes added to the end."
   (vlf-verify-size)
+  (if (derived-mode-p 'hexl-mode)
+      (setq start (- start (mod start hexl-bits))
+            end (+ end (- hexl-bits (mod end hexl-bits)))))
   (cond ((or (<= end start) (<= end 0)
              (<= vlf-file-size start))
          (when (or (not (buffer-modified-p))
@@ -133,6 +136,7 @@ bytes added to the end."
                      vlf-end-pos))
          (shifts
           (cond
+           ((and hexl (not modified)) (vlf-move-to-chunk-2 start end))
            ((or (< edit-end start) (< end vlf-start-pos)
                 (not (verify-visited-file-modtime (current-buffer))))
             (when (or (not modified)
@@ -233,20 +237,21 @@ bytes added to the end."
   (vlf-verify-size t)
   (setq vlf-start-pos (max 0 start)
         vlf-end-pos (min end vlf-file-size))
-  (let (shifts)
+  (let ((shifts '(0 . 0)))
     (let ((inhibit-read-only t)
           (pos (position-bytes (point))))
       (vlf-with-undo-disabled
-       (let ((hexl (derived-mode-p 'hexl-mode)))
-         (if hexl (hexl-mode-exit t))
        (erase-buffer)
+       (if (derived-mode-p 'hexl-mode)
+           (progn (vlf-tune-insert-file-contents-literally
+                   vlf-start-pos vlf-end-pos)
+                  (vlf-tune-hexlify))
          (setq shifts (vlf-insert-file-contents vlf-start-pos
                                                 vlf-end-pos t t)
                vlf-start-pos (- vlf-start-pos (car shifts))
-               vlf-end-pos (+ vlf-end-pos (cdr shifts)))
+               vlf-end-pos (+ vlf-end-pos (cdr shifts)))))
-         (if hexl (vlf-tune-hexlify)))
       (goto-char (or (byte-to-position (+ pos (car shifts)))
-                      (point-max)))))
+                     (point-max))))
     (set-buffer-modified-p nil)
     (or (eq buffer-undo-list t)
         (setq buffer-undo-list nil))

vlf-occur.el

@@ -154,7 +154,7 @@ Prematurely ending indexing will still show what's found so far."
         (insert-bps vlf-tune-insert-bps)
         (encode-bps vlf-tune-encode-bps)
         (hexl-bps vlf-tune-hexl-bps)
-        (dehexlify-bps vlf-tune-dehexlify-bps))
+        (insert-raw-bps vlf-tune-insert-raw-bps))
     (with-temp-buffer
       (setq buffer-file-name file
             buffer-file-truename file
@@ -166,26 +166,25 @@ Prematurely ending indexing will still show what's found so far."
               vlf-tune-encode-bps encode-bps)
         (if is-hexl
             (progn (setq vlf-tune-hexl-bps hexl-bps
-                         vlf-tune-dehexlify-bps dehexlify-bps)
+                         vlf-tune-insert-raw-bps insert-raw-bps)
-                   (vlf-tune-batch '(:hexl :dehexlify :insert :encode)))
+                   (vlf-tune-batch '(:hexl :raw)))
           (vlf-tune-batch '(:insert :encode))))
       (vlf-mode 1)
-      (if is-hexl (vlf-tune-hexlify))
+      (if is-hexl (hexl-mode))
       (goto-char (point-min))
-      (vlf-with-undo-disabled
+      (vlf-build-occur regexp vlf-buffer)
-       (vlf-build-occur regexp vlf-buffer))
       (when vlf-tune-enabled
         (setq insert-bps vlf-tune-insert-bps
               encode-bps vlf-tune-encode-bps)
         (if is-hexl
-            (setq insert-bps vlf-tune-insert-bps
+            (setq hexl-bps vlf-tune-hexl-bps
-                  encode-bps vlf-tune-encode-bps))))
+                  insert-raw-bps vlf-tune-insert-raw-bps))))
     (when vlf-tune-enabled              ;merge back tune measurements
       (setq vlf-tune-insert-bps insert-bps
             vlf-tune-encode-bps encode-bps)
       (if is-hexl
-          (setq vlf-tune-insert-bps insert-bps
+          (setq vlf-tune-hexl-bps hexl-bps
-                vlf-tune-encode-bps encode-bps)))))
+                vlf-tune-insert-raw-bps insert-raw-bps)))))
 
 (defun vlf-occur (regexp)
   "Make whole file occur style index for REGEXP.
@@ -203,7 +202,7 @@ Prematurely ending indexing will still show what's found so far."
           (batch-size vlf-batch-size)
           (is-hexl (derived-mode-p 'hexl-mode)))
       (vlf-tune-batch (if (derived-mode-p 'hexl-mode)
-                          '(:hexl :dehexlify :insert :encode)
+                          '(:hexl :raw)
                         '(:insert :encode)))
       (vlf-with-undo-disabled
        (vlf-move-to-batch 0)
@@ -235,7 +234,7 @@ Prematurely ending indexing will still show what's found so far."
          (is-hexl (derived-mode-p 'hexl-mode))
          (end-of-file nil)
          (time (float-time))
-         (tune-types (if is-hexl '(:hexl :dehexlify :insert :encode)
+         (tune-types (if is-hexl '(:hexl :raw)
                        '(:insert :encode)))
          (reporter (make-progress-reporter
                     (concat "Building index for " regexp "...")

vlf-search.el

@@ -54,7 +54,7 @@ Return t if search has been at least partially successful."
          (match-end-pos match-start-pos)
          (to-find count)
          (is-hexl (derived-mode-p 'hexl-mode))
-         (tune-types (if is-hexl '(:hexl :dehexlify :insert :encode)
+         (tune-types (if is-hexl '(:hexl :raw)
                        '(:insert :encode)))
          (font-lock font-lock-mode))
     (font-lock-mode 0)
@@ -160,7 +160,7 @@ Return nil if nothing found."
         (vlf-move-to-chunk match-chunk-start match-chunk-end))
       (setq vlf-batch-size (vlf-tune-optimal-load
                             (if (derived-mode-p 'hexl-mode)
-                                '(:hexl :dehexlify :insert :encode)
+                                '(:hexl :raw)
                               '(:insert :encode))))
       (let* ((match-end (or (byte-to-position (- match-pos-end
                                                  vlf-start-pos))

vlf-tune.el

@@ -174,17 +174,17 @@ SIZE is number of bytes that are saved."
 
 (defun vlf-tune-hexlify ()
   "Activate `hexl-mode' and save time it takes."
-  (or (derived-mode-p 'hexl-mode)
+  (let ((time (car (vlf-time (hexlify-buffer)))))
-      (let ((time (car (vlf-time (hexl-mode)))))
+    (setq hexl-max-address (+ (* (/ (1- (buffer-size))
+                                    (hexl-line-displen)) 16) 15))
     (vlf-tune-add-measurement vlf-tune-hexl-bps
-                                  hexl-max-address time))))
+                              hexl-max-address time)))
 
 (defun vlf-tune-dehexlify ()
   "Exit `hexl-mode' and save time it takes."
-  (if (derived-mode-p 'hexl-mode)
+  (let ((time (car (vlf-time (dehexlify-buffer)))))
-      (let ((time (car (vlf-time (hexl-mode-exit)))))
     (vlf-tune-add-measurement vlf-tune-dehexlify-bps
-                                  hexl-max-address time))))
+                              hexl-max-address time)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; tuning
@@ -233,7 +233,8 @@ unless DONT-APPROXIMATE is t."
                      `(aset ,vec ,index
                             (vlf-tune-approximate-nearby ,vec ,index))
                    `(vlf-tune-approximate-nearby ,vec ,index)))
-               (t val)))))
+               (t val)))
+     most-positive-fixnum))
 
 (defmacro vlf-tune-get-vector (key)
   "Get vlf-tune vector corresponding to KEY."
@@ -268,7 +269,7 @@ If it is number, stop as soon as cumulative time gets equal or above."
         (let ((bps (if (consp el)
                        (vlf-tune-assess (car el) (cadr el) index
                                         approximate)
-                     (vlf-tune-assess el 1 index approximate))))
+                     (vlf-tune-assess el 1.0 index approximate))))
           (if (zerop bps)
               (throw 'result nil)
             (setq time (+ time (/ size bps)))
@@ -332,7 +333,7 @@ MIN and MAX specify interval of indexes to search."
           (setq vlf-batch-size (* (1+ left-idx) vlf-tune-step)))))))
 
 (defun vlf-tune-linear (types max-idx)
-  "Adjust `vlf-batch-size' to optimal value using linear search, \
+  "Adjust `vlf-batch-size' to optimal value using linear search,\
 optimizing over TYPES up to MAX-IDX."
   (let ((best-idx 0)
         (best-bps 0)
@@ -346,7 +347,7 @@ optimizing over TYPES up to MAX-IDX."
               ((< best-bps bps) (setq best-idx idx
                                       best-bps bps))))
       (setq idx (1+ idx)))
-    (or (not none-missing)
+    (if none-missing
         (setq vlf-batch-size (* (1+ best-idx) vlf-tune-step)))))
 
 (defun vlf-tune-batch (types &optional linear)
@@ -376,11 +377,12 @@ Best considered where primitive operations total is closest to
 confine search to this region."
   (if vlf-tune-enabled
       (progn
-        (setq max-idx (min (or max-idx vlf-tune-max)
+        (setq min-idx (max 0 (or min-idx 0))
+              max-idx (min (or max-idx vlf-tune-max)
                            (1- (/ (min vlf-tune-max
                                        (/ (1+ vlf-file-size) 2))
                                   vlf-tune-step))))
-        (let* ((idx (max 0 (or min-idx 0)))
+        (let* ((idx min-idx)
                (best-idx idx)
                (best-time-diff vlf-tune-load-time)
                (all-less t)

vlf.el

@@ -2,7 +2,7 @@
 
 ;; Copyright (C) 2006, 2012-2014 Free Software Foundation, Inc.
 
-;; Version: 1.6
+;; Version: 1.7
 ;; Keywords: large files, utilities
 ;; Maintainer: Andrey Kotlarski <m00naticus@gmail.com>
 ;; Authors: 2006 Mathias Dahl <mathias.dahl@gmail.com>
@@ -125,13 +125,23 @@ values are: `write', `ediff', `occur', `search', `goto-line'."
          (remove-hook 'write-file-functions 'vlf-write t)
          (remove-hook 'after-change-major-mode-hook
                       'vlf-keep-alive t)
-         (let ((hexl (derived-mode-p 'hexl-mode)))
+         (if (derived-mode-p 'hexl-mode)
-           (if hexl (hexl-mode-exit))
+             (let ((line (/ (1+ vlf-start-pos) hexl-bits))
+                   (pos (point)))
+               (if (consp buffer-undo-list)
+                   (setq buffer-undo-list nil))
+               (vlf-with-undo-disabled
+                (insert-file-contents-literally buffer-file-name
+                                                t nil nil t)
+                (hexlify-buffer))
+               (set-buffer-modified-p nil)
+               (goto-char (point-min))
+               (forward-line line)
+               (forward-char pos))
            (let ((pos (+ vlf-start-pos (position-bytes (point)))))
              (vlf-with-undo-disabled
               (insert-file-contents buffer-file-name t nil nil t))
-             (goto-char (byte-to-position pos)))
+             (goto-char (byte-to-position pos))))
-           (if hexl (hexl-mode)))
          (rename-buffer (file-name-nondirectory buffer-file-name) t))
         (t (setq vlf-mode t))))
 
@@ -170,7 +180,7 @@ When prefix argument is negative
   (interactive "p")
   (vlf-verify-size)
   (vlf-tune-load (if (derived-mode-p 'hexl-mode)
-                     '(:hexl :dehexlify :insert :encode)
+                     '(:hexl :raw)
                    '(:insert :encode)))
   (let* ((end (min (+ vlf-end-pos (* vlf-batch-size (abs append)))
                    vlf-file-size))
@@ -189,7 +199,7 @@ When prefix argument is negative
   (if (zerop vlf-start-pos)
       (error "Already at BOF"))
   (vlf-tune-load (if (derived-mode-p 'hexl-mode)
-                     '(:hexl :dehexlify :insert :encode)
+                     '(:hexl :raw)
                    '(:insert :encode)))
   (let* ((start (max 0 (- vlf-start-pos (* vlf-batch-size (abs prepend)))))
          (end (if (< prepend 0)
@@ -241,6 +251,16 @@ When prefix argument is negative
        (if (and vlf-mode (pos-visible-in-window-p (point-min)))
            (progn (vlf-prev-batch 1)
                   (goto-char (point-max)))
+         ad-do-it))
+
+     (defadvice hexl-mode-exit (around vlf-hexl-mode-exit
+                                       activate compile)
+       "Exit `hexl-mode' gracefully in case `vlf-mode' is active."
+       (if (and vlf-mode (not (buffer-modified-p)))
+           (vlf-with-undo-disabled
+            (erase-buffer)
+            ad-do-it
+            (vlf-move-to-chunk-2 vlf-start-pos vlf-end-pos))
          ad-do-it))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -260,7 +280,7 @@ with the prefix argument DECREASE it is halved."
    (list (read-number "Size in bytes: "
                       (vlf-tune-optimal-load
                        (if (derived-mode-p 'hexl-mode)
-                           '(:hexl :dehexlify :insert :encode)
+                           '(:hexl :raw)
                          '(:insert :encode))))))
   (setq vlf-batch-size size)
   (vlf-move-to-batch vlf-start-pos))
@@ -269,7 +289,7 @@ with the prefix argument DECREASE it is halved."
   "Jump to beginning of file content."
   (interactive)
   (vlf-tune-load (if (derived-mode-p 'hexl-mode)
-                     '(:hexl :dehexlify :insert :encode)
+                     '(:hexl :raw)
                    '(:insert :encode)))
   (vlf-move-to-batch 0))
 
@@ -278,7 +298,7 @@ with the prefix argument DECREASE it is halved."
   (interactive)
   (vlf-verify-size)
   (vlf-tune-load (if (derived-mode-p 'hexl-mode)
-                     '(:hexl :dehexlify :insert :encode)
+                     '(:hexl :raw)
                    '(:insert :encode)))
   (vlf-move-to-batch vlf-file-size))
 
@@ -296,7 +316,7 @@ Ask for confirmation if NOCONFIRM is nil."
   "Go to to chunk N."
   (interactive "nGoto to chunk: ")
   (vlf-tune-load (if (derived-mode-p 'hexl-mode)
-                     '(:hexl :dehexlify :insert :encode)
+                     '(:hexl :raw)
                    '(:insert :encode)))
   (vlf-move-to-batch (* (1- n) vlf-batch-size)))