Bump version and update README.

saving.

README.org

@@ -1,22 +1,105 @@
 * View Large Files Improved
 
-An Emacs mode that allows viewing files in chunks.  This is a fork
-that builds on the bare bones GNU ELPA vlf.el.  It adds the following
-improvements:
+An Emacs mode that allows viewing, editing and searching in large
+files in chunks.  Batch size can be adjusted on the fly and bounds the
+memory that is to be used for operations on the file.
 
+This mode builds on the bare bones GNU ELPA vlf.el.  It adds the
+following improvements:
+
+- proper dealing with Unicode
 - regular expression search on whole file (in constant memory
   determined by current batch size)
 - chunk editing (if size has changed, saving is done in constant
   memory determined by current batch size)
+- occur like indexing
 - options to jump to beginning, end or arbitrary file chunk
 - ability to jump/insert given number of batches at once
 - newly added content is acknowledged if file has changed size
   meanwhile
-- vlfi is added as an option when opening large files
+- automatic scrolling of batches
+- VLFI is added as an option when opening large files
 
-* Known issues
+GNU Emacs 23 and 24 are supported.
 
-- When chunk starts with only part of multi-byte character, encoding
-  for this chunk is clobbered visibly, don't edit/save in such case.
-  Workaround is to move back to the first non-clobbered chunk and
-  enlarge batch size until this chunk comes into range.
+* Overview and tips
+
+M-x vlfi PATH-TO-FILE
+
+** Unicode
+
+Emacs' Unicode support is leveraged so you'll not see bare bytes but
+characters decoded as if file is normally opened.  This holds for
+editing, search and indexing.
+
+** 32-bit GNU Emacs
+
+Regular Emacs integers are used, so if you use 32-bit Emacs without
+bignum support and have really huge file (with size beyond the maximum
+integer value), VLFI will probably not quite work.
+
+** Memory control
+
+*vlfi-batch-size* bounds the memory used for all operations.
+
+** Special mode
+
+VLFI is derived from special-mode and keeps all its properties.  For
+example you can directly press digits to enter prefix arguments.
+
+** Changing major mode
+
+You can (temporarily) change major mode to whatever you like (for
+example hexl-mode).  Saving will insert contents as intended.  You can
+return to *vlfi-mode* too.
+
+* Detail usage
+
+** Controlling batch size
+
+*+* and *-* control current batch size by factors of 2.
+
+You can also set by hand local variable *vlfi-batch-size* and then
+refresh with *g*.
+
+** Move around
+
+*M-PgUp* and *M-PgDn* move chunk by chunk.  With positive prefix
+argument they move prefix number of batches.  With negative - append
+prefix number of batches.
+
+*[* and *]* take you to the beginning and end of file respectively.
+
+*j* jumps to given chunk.  To see where you are in file and how many chunks
+there are (using the current batch size), look at the bracketed part
+of the buffer name, batch size is also there - at the end.
+
+** Search whole file
+
+*s* and *r* search forward and backward respectively over the whole
+file.  This is done chunk by chunk so if you have really huge file -
+you'd better set somewhat bigger batch size beforehand.
+
+** Occur over whole file
+
+*o* builds index for given regular expression just like occur-mode.
+It does this chunk by chunk over the whole file.  Note that even if
+you prematurely stop it with *C-g*, it will still show index of what's
+found so far.
+
+** Jump to line
+
+*l* jumps to given line in file.  This is done by searching from the
+beginning, so again the bigger current batch size, the quicker.  With
+negative argument, lines are counted from the end of file.
+
+** Edit
+
+*e* enters VLFI in edit mode.  If editing doesn't change size of
+the chunk, only this chunk is saved.  Otherwise the remaining part of
+the file is adjusted chunk by chunk, so again you'd better have bigger
+current batch size.  If chunk has been expanded the memory used is
+
+#+BEGIN_EXAMPLE
+(batch size + difference to the original chunk size) x 2
+#+END_EXAMPLE

vlfi.el

@@ -2,7 +2,7 @@
 
 ;; Copyright (C) 2006, 2012, 2013  Free Software Foundation, Inc.
 
-;; Version: 0.6
+;; Version: 0.9
 ;; Keywords: large files, utilities
 ;; Authors: 2006 Mathias Dahl <mathias.dahl@gmail.com>
 ;;          2012 Sam Steingold <sds@gnu.org>
@@ -26,11 +26,11 @@
 ;;; Commentary:
 
 ;; This package provides the M-x vlfi command, which visits part of a
-;; large file in a read-only buffer without visiting the entire file.
+;; large file without loading the entire file.
 ;; The buffer uses VLFI mode, which defines several commands for
-;; moving around, searching and editing selected chunk of file.
+;; moving around, searching and editing selected part of file.
 
-;; This package is an improved fork of the vlf.el package.
+;; This package is upgraded version of the vlf.el package.
 
 ;;; Code:
 
@@ -47,8 +47,7 @@
 ;;; Keep track of file position.
 (defvar vlfi-start-pos 0
   "Absolute position of the visible chunk start.")
-(defvar vlfi-end-pos vlfi-batch-size
-  "Absolute position of the visible chunk end.")
+(defvar vlfi-end-pos 0 "Absolute position of the visible chunk end.")
 (defvar vlfi-file-size 0 "Total size of presented file.")
 
 (defvar vlfi-mode-map
@@ -62,10 +61,12 @@
         (vlfi-change-batch-size t)))
     (define-key map "s" 'vlfi-re-search-forward)
     (define-key map "r" 'vlfi-re-search-backward)
+    (define-key map "o" 'vlfi-occur)
     (define-key map "[" 'vlfi-beginning-of-file)
     (define-key map "]" 'vlfi-end-of-file)
     (define-key map "e" 'vlfi-edit-mode)
     (define-key map "j" 'vlfi-jump-to-chunk)
+    (define-key map "l" 'vlfi-goto-line)
     map)
   "Keymap for `vlfi-mode'.")
 
@@ -74,7 +75,9 @@
   (setq buffer-read-only t)
   (set-buffer-modified-p nil)
   (buffer-disable-undo)
-  (add-hook 'write-contents-functions 'vlfi-write)
+  (make-local-variable 'write-file-functions)
+  (add-hook 'write-file-functions 'vlfi-write)
+  (make-local-variable 'revert-buffer-function)
   (setq revert-buffer-function 'vlfi-revert)
   (make-local-variable 'vlfi-batch-size)
   (put 'vlfi-batch-size 'permanent-local t)
@@ -86,28 +89,27 @@
   (put 'vlfi-file-size 'permanent-local t))
 
 ;;;###autoload
-(defun vlfi (file &optional from-end)
-  "View Large FILE.  With FROM-END prefix, view from the back.
+(defun vlfi (file)
+  "View Large FILE.
 Batches of the file data from FILE will be displayed in a read-only
 buffer.  You can customize number of bytes displayed by customizing
 `vlfi-batch-size'."
-  (interactive "fFile to open: \nP")
+  (interactive "fFile to open: ")
   (with-current-buffer (generate-new-buffer "*vlfi*")
-    (setq buffer-file-name file
-          vlfi-file-size (nth 7 (file-attributes file)))
-    (vlfi-insert-file from-end)
     (vlfi-mode)
+    (setq buffer-file-name file
+          vlfi-file-size (vlfi-get-file-size file))
+    (vlfi-insert-file)
     (switch-to-buffer (current-buffer))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; integration with other packages
 
 ;;;###autoload
-(defun dired-vlfi (from-end)
-  "In Dired, visit the file on this line in VLFI mode.
-With FROM-END prefix, view from the back."
-  (interactive "P")
-  (vlfi (dired-get-file-for-visit) from-end))
+(defun dired-vlfi ()
+  "In Dired, visit the file on this line in VLFI mode."
+  (interactive)
+  (vlfi (dired-get-file-for-visit)))
 
 ;;;###autoload
 (eval-after-load "dired"
@@ -136,7 +138,7 @@ OP-TYPE specifies the file operation being performed over FILENAME."
                            '(?o ?O ?v ?V ?a ?A))))
          (cond ((memq char '(?o ?O)))
                ((memq char '(?v ?V))
-                (vlfi filename nil)
+                (vlfi filename)
                 (error ""))
                ((memq char '(?a ?A))
                 (error "Aborted"))))))
@@ -145,11 +147,30 @@ OP-TYPE specifies the file operation being performed over FILENAME."
 ;;;###autoload
 (fset 'abort-if-file-too-large 'vlfi-if-file-too-large)
 
-;; non recent Emacs
+;; scroll auto batching
+(defadvice scroll-up (around vlfi-scroll-up
+                             activate compile)
+  "Slide to next batch if at end of buffer in `vlfi-mode'."
+  (if (and (eq major-mode 'vlfi-mode)
+           (eobp))
+      (progn (vlfi-next-batch 1)
+             (goto-char (point-min)))
+    ad-do-it))
+
+(defadvice scroll-down (around vlfi-scroll-down
+                               activate compile)
+  "Slide to previous batch if at beginning of buffer  in `vlfi-mode'."
+  (if (and (eq major-mode 'vlfi-mode)
+           (bobp))
+      (progn (vlfi-prev-batch 1)
+             (goto-char (point-max)))
+    ad-do-it))
+
+;; non-recent Emacs
 (unless (fboundp 'file-size-human-readable)
   (defun file-size-human-readable (file-size)
     "Print FILE-SIZE in MB."
-    (format "%.1fMB" (/ file-size 1024.0))))
+    (format "%.1fMB" (/ file-size 1048576.0))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; utilities
@@ -179,6 +200,10 @@ with the prefix argument DECREASE it is halved."
   "Update the current buffer name."
   (rename-buffer (vlfi-format-buffer-name) t))
 
+(defun vlfi-get-file-size (file)
+  "Get size in bytes of FILE."
+  (nth 7 (file-attributes file)))
+
 (defun vlfi-insert-file (&optional from-end)
   "Insert first chunk of current file contents in current buffer.
 With FROM-END prefix, start from the back."
@@ -199,10 +224,14 @@ With FROM-END prefix, start from the back."
   (interactive)
   (vlfi-insert-file t))
 
-(defun vlfi-revert (&rest args)
-  "Revert current chunk.  Ignore ARGS."
-  (ignore args)
-  (vlfi-move-to-chunk vlfi-start-pos vlfi-end-pos))
+(defun vlfi-revert (&optional ignore-auto noconfirm)
+  "Revert current chunk.  Ignore IGNORE-AUTO.
+Ask for confirmation if NOCONFIRM is nil."
+  (ignore ignore-auto)
+  (or noconfirm
+      (yes-or-no-p (format "Revert buffer from file %s? "
+                           buffer-file-name))
+      (vlfi-move-to-chunk vlfi-start-pos vlfi-end-pos)))
 
 (defun vlfi-jump-to-chunk (n)
   "Go to to chunk N."
@@ -219,17 +248,14 @@ When prefix argument is supplied and positive
 When prefix argument is negative
  append next APPEND number of batches to the existing buffer."
   (interactive "p")
-  (let ((end (+ vlfi-end-pos (* vlfi-batch-size
-                                (abs append)))))
-    (when (< vlfi-file-size end)        ; re-check file size
-      (setq vlfi-file-size (nth 7 (file-attributes buffer-file-name)))
-      (cond ((= vlfi-end-pos vlfi-file-size)
-             (error "Already at EOF"))
-            ((< vlfi-file-size end)
-             (setq end vlfi-file-size))))
+  (or (verify-visited-file-modtime (current-buffer))
+      (setq vlfi-file-size (vlfi-get-file-size buffer-file-name)))
+  (let ((end (min (+ vlfi-end-pos (* vlfi-batch-size
+                                     (abs append)))
+                  vlfi-file-size)))
     (let ((inhibit-read-only t)
           (do-append (< append 0))
-          (pos (point)))
+          (pos (position-bytes (point))))
       (if do-append
           (goto-char (point-max))
         (setq vlfi-start-pos (- end vlfi-batch-size))
@@ -238,8 +264,9 @@ When prefix argument is negative
                                                      vlfi-end-pos
                                                    vlfi-start-pos)
                             end)
-      (goto-char pos))
-    (setq vlfi-end-pos end))
+      (setq vlfi-end-pos end)
+      (goto-char (or (byte-to-position (+ pos (vlfi-adjust-chunk)))
+                     (point-max)))))
   (set-visited-file-modtime)
   (set-buffer-modified-p nil)
   (vlfi-update-buffer-name))
@@ -253,87 +280,125 @@ When prefix argument is negative
   (interactive "p")
   (if (zerop vlfi-start-pos)
       (error "Already at BOF"))
+  (or (verify-visited-file-modtime (current-buffer))
+      (setq vlfi-file-size (vlfi-get-file-size buffer-file-name)))
   (let ((inhibit-read-only t)
         (start (max 0 (- vlfi-start-pos (* vlfi-batch-size
                                            (abs prepend)))))
         (do-prepend (< prepend 0))
-        (pos (- (point-max) (point))))
+        (pos (- (position-bytes (point-max))
+                (position-bytes (point)))))
     (if do-prepend
         (goto-char (point-min))
-      (setq vlfi-end-pos (+ start vlfi-batch-size))
+      (setq vlfi-end-pos (min (+ start vlfi-batch-size)
+                              vlfi-file-size))
       (erase-buffer))
     (insert-file-contents buffer-file-name nil start
                           (if do-prepend
                               vlfi-start-pos
                             vlfi-end-pos))
-    (goto-char (- (point-max) pos))
-    (setq vlfi-start-pos start))
+    (setq vlfi-start-pos start)
+    (setq pos (+ pos (vlfi-adjust-chunk)))
+    (goto-char (or (byte-to-position (- (position-bytes (point-max))
+                                        pos))
+                   (point-max))))
   (set-visited-file-modtime)
   (set-buffer-modified-p nil)
   (vlfi-update-buffer-name))
 
-(defun vlfi-move-to-batch (start)
+(defun vlfi-move-to-batch (start &optional minimal)
   "Move to batch determined by START.
-Adjust according to file start/end and show `vlfi-batch-size' bytes."
+Adjust according to file start/end and show `vlfi-batch-size' bytes.
+When given MINIMAL flag, skip non important operations."
+  (or (verify-visited-file-modtime (current-buffer))
+      (setq vlfi-file-size (vlfi-get-file-size buffer-file-name)))
   (setq vlfi-start-pos (max 0 start)
-        vlfi-end-pos (+ vlfi-start-pos vlfi-batch-size))
-  (if (< vlfi-file-size vlfi-end-pos)   ; re-check file size
-      (setq vlfi-file-size
-            (nth 7 (file-attributes buffer-file-name))
-            vlfi-end-pos (min vlfi-end-pos vlfi-file-size)
-            vlfi-start-pos (max 0 (- vlfi-end-pos vlfi-batch-size))))
+        vlfi-end-pos (min (+ vlfi-start-pos vlfi-batch-size)
+                          vlfi-file-size))
+  (if (= vlfi-file-size vlfi-end-pos)   ; re-check file size
+      (setq vlfi-start-pos (max 0 (- vlfi-end-pos vlfi-batch-size))))
   (let ((inhibit-read-only t)
-        (pos (point)))
+        (pos (position-bytes (point))))
     (erase-buffer)
     (insert-file-contents buffer-file-name nil
                           vlfi-start-pos vlfi-end-pos)
-    (goto-char pos))
-  (set-visited-file-modtime)
+    (goto-char (or (byte-to-position (+ pos (vlfi-adjust-chunk)))
+                   (point-max))))
   (set-buffer-modified-p nil)
-  (vlfi-update-buffer-name))
+  (set-visited-file-modtime)
+  (or minimal(vlfi-update-buffer-name)))
 
-(defun vlfi-move-to-chunk (start end)
-  "Move to chunk determined by START END."
-  (if (< vlfi-file-size end)            ; re-check file size
-      (setq vlfi-file-size (nth 7
-                                (file-attributes buffer-file-name))))
+(defun vlfi-move-to-chunk (start end &optional minimal)
+  "Move to chunk determined by START END.
+When given MINIMAL flag, skip non important operations."
+  (or (verify-visited-file-modtime (current-buffer))
+      (setq vlfi-file-size (vlfi-get-file-size buffer-file-name)))
   (setq vlfi-start-pos (max 0 start)
         vlfi-end-pos (min end vlfi-file-size))
   (let ((inhibit-read-only t)
-        (pos (point)))
+        (pos (position-bytes (point))))
     (erase-buffer)
     (insert-file-contents buffer-file-name nil
                           vlfi-start-pos vlfi-end-pos)
-    (goto-char pos))
-  (set-visited-file-modtime)
+    (goto-char (or (byte-to-position (+ pos (vlfi-adjust-chunk)))
+                   (point-max))))
   (set-buffer-modified-p nil)
-  (vlfi-update-buffer-name))
+  (set-visited-file-modtime)
+  (or minimal (vlfi-update-buffer-name)))
+
+(defun vlfi-adjust-chunk ()
+  "Adjust chunk beginning until content can be properly decoded.
+Return number of bytes moved back for this to happen."
+  (let ((shift 0)
+        (chunk-size (- vlfi-end-pos vlfi-start-pos)))
+    (while (and (not (zerop vlfi-start-pos))
+                (< shift 4)
+                (/= chunk-size
+                    (length (encode-coding-region
+                             (point-min) (point-max)
+                             buffer-file-coding-system t))))
+      (setq shift (1+ shift)
+            vlfi-start-pos (1- vlfi-start-pos)
+            chunk-size (1+ chunk-size))
+      (let ((inhibit-read-only t))
+        (erase-buffer)
+        (insert-file-contents buffer-file-name nil
+                              vlfi-start-pos vlfi-end-pos)))
+    (set-buffer-modified-p nil)
+    shift))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; search
 
-(defun vlfi-re-search (regexp count backward)
-  "Search for REGEXP COUNT number of times forward or BACKWARD."
-  (let* ((match-start-pos (+ vlfi-start-pos (point)))
+(defun vlfi-re-search (regexp count backward batch-step)
+  "Search for REGEXP COUNT number of times forward or BACKWARD.
+BATCH-STEP is amount of overlap between successive chunks."
+  (assert (< 0 count))
+  (let* ((match-chunk-start vlfi-start-pos)
+         (match-chunk-end vlfi-end-pos)
+         (match-start-pos (+ vlfi-start-pos (position-bytes (point))))
          (match-end-pos match-start-pos)
          (to-find count)
-         (search-reporter (make-progress-reporter
-                           (concat "Searching for " regexp)
-                           (if backward
-                               (- vlfi-file-size vlfi-end-pos)
-                             vlfi-start-pos)
-                           vlfi-file-size))
-         (batch-step (/ vlfi-batch-size 8))) ; amount of chunk overlap
+         (reporter (make-progress-reporter
+                    (concat "Searching for " regexp "...")
+                    (if backward
+                        (- vlfi-file-size vlfi-end-pos)
+                      vlfi-start-pos)
+                    vlfi-file-size)))
     (unwind-protect
         (catch 'end-of-file
           (if backward
               (while (not (zerop to-find))
                 (cond ((re-search-backward regexp nil t)
                        (setq to-find (1- to-find)
+                             match-chunk-start vlfi-start-pos
+                             match-chunk-end vlfi-end-pos
                              match-start-pos (+ vlfi-start-pos
-                                                (match-beginning 0))
+                                                (position-bytes
+                                                 (match-beginning 0)))
                              match-end-pos (+ vlfi-start-pos
-                                              (match-end 0))))
+                                              (position-bytes
+                                               (match-end 0)))))
                       ((zerop vlfi-start-pos)
                        (throw 'end-of-file nil))
                       (t (let ((batch-move (- vlfi-start-pos
@@ -342,82 +407,322 @@ Adjust according to file start/end and show `vlfi-batch-size' bytes."
                            (vlfi-move-to-batch
                             (if (< match-start-pos batch-move)
                                 (- match-start-pos vlfi-batch-size)
-                              batch-move)))
+                              batch-move) t))
                          (goto-char (if (< match-start-pos
                                            vlfi-end-pos)
-                                        (- match-start-pos
-                                           vlfi-start-pos)
+                                        (or (byte-to-position
+                                             (- match-start-pos
+                                                vlfi-start-pos))
+                                            (point-max))
                                       (point-max)))
-                         (progress-reporter-update search-reporter
-                                                   vlfi-start-pos))))
+                         (progress-reporter-update
+                          reporter (- vlfi-file-size
+                                      vlfi-start-pos)))))
             (while (not (zerop to-find))
               (cond ((re-search-forward regexp nil t)
                      (setq to-find (1- to-find)
+                           match-chunk-start vlfi-start-pos
+                           match-chunk-end vlfi-end-pos
                            match-start-pos (+ vlfi-start-pos
-                                              (match-beginning 0))
+                                              (position-bytes
+                                               (match-beginning 0)))
                            match-end-pos (+ vlfi-start-pos
-                                            (match-end 0))))
+                                            (position-bytes
+                                             (match-end 0)))))
                     ((= vlfi-end-pos vlfi-file-size)
                      (throw 'end-of-file nil))
                     (t (let ((batch-move (- vlfi-end-pos batch-step)))
                          (vlfi-move-to-batch
                           (if (< batch-move match-end-pos)
                               match-end-pos
-                            batch-move)))
+                            batch-move) t))
                        (goto-char (if (< vlfi-start-pos match-end-pos)
-                                      (- match-end-pos vlfi-start-pos)
+                                      (or (byte-to-position
+                                           (- match-end-pos
+                                              vlfi-start-pos))
+                                          (point-min))
                                     (point-min)))
-                       (progress-reporter-update search-reporter
+                       (progress-reporter-update reporter
                                                  vlfi-end-pos)))))
-          (progress-reporter-done search-reporter))
+          (progress-reporter-done reporter))
       (if backward
-          (vlfi-goto-match match-end-pos match-start-pos
+          (vlfi-goto-match match-chunk-start match-chunk-end
+                           match-end-pos match-start-pos
                            count to-find)
-        (vlfi-goto-match match-start-pos match-end-pos
+        (vlfi-goto-match match-chunk-start match-chunk-end
+                         match-start-pos match-end-pos
                          count to-find)))))
 
-(defun vlfi-goto-match (match-pos-start match-pos-end count to-find)
-  "Move to chunk surrounding MATCH-POS-START and MATCH-POS-END.
+(defun vlfi-goto-match (match-chunk-start match-chunk-end
+                                          match-pos-start
+                                          match-pos-end
+                                          count to-find)
+  "Move to MATCH-CHUNK-START MATCH-CHUNK-END surrounding \
+MATCH-POS-START and MATCH-POS-END.
 According to COUNT and left TO-FIND, show if search has been
 successful.  Return nil if nothing found."
-  (let ((success (zerop to-find)))
-    (or success
-        (vlfi-move-to-batch (- match-pos-start
-                               (/ vlfi-batch-size 2))))
-    (let* ((match-end (- match-pos-end vlfi-start-pos))
-           (overlay (make-overlay (- match-pos-start vlfi-start-pos)
-                                  match-end)))
-      (overlay-put overlay 'face 'region)
-      (or success (goto-char match-end))
-      (prog1 (cond (success t)
-                   ((< to-find count)
-                    (message "Moved to the %d match which is last"
-                             (- count to-find))
-                    t)
-                   (t (message "Not found")
-                      nil))
+  (if (= count to-find)
+      (progn (vlfi-move-to-chunk match-chunk-start match-chunk-end)
+             (goto-char (or (byte-to-position (- match-pos-start
+                                                 vlfi-start-pos))
+                            (point-max)))
+             (message "Not found")
+             nil)
+    (let ((success (zerop to-find)))
+      (if success
+          (vlfi-update-buffer-name)
+        (vlfi-move-to-chunk match-chunk-start match-chunk-end))
+      (let* ((match-end (or (byte-to-position (- match-pos-end
+                                                 vlfi-start-pos))
+                            (point-max)))
+             (overlay (make-overlay (byte-to-position
+                                     (- match-pos-start
+                                        vlfi-start-pos))
+                                    match-end)))
+        (overlay-put overlay 'face 'match)
+        (unless success
+          (goto-char match-end)
+          (message "Moved to the %d match which is last"
+                   (- count to-find)))
         (sit-for 0.1)
-        (delete-overlay overlay)))))
+        (delete-overlay overlay)
+        t))))
 
 (defun vlfi-re-search-forward (regexp count)
   "Search forward for REGEXP prefix COUNT number of times.
 Search is performed chunk by chunk in `vlfi-batch-size' memory."
   (interactive (list (read-regexp "Search whole file"
                                   (if regexp-history
-                                      (car regexp-history))
-                                  'regexp-history)
+                                      (car regexp-history)))
                      (or current-prefix-arg 1)))
-  (vlfi-re-search regexp count nil))
+  (vlfi-re-search regexp count nil (/ vlfi-batch-size 8)))
 
 (defun vlfi-re-search-backward (regexp count)
   "Search backward for REGEXP prefix COUNT number of times.
 Search is performed chunk by chunk in `vlfi-batch-size' memory."
   (interactive (list (read-regexp "Search whole file backward"
                                   (if regexp-history
-                                      (car regexp-history))
-                                  'regexp-history)
+                                      (car regexp-history)))
                      (or current-prefix-arg 1)))
-  (vlfi-re-search regexp count t))
+  (vlfi-re-search regexp count t (/ vlfi-batch-size 8)))
+
+(defun vlfi-goto-line (n)
+  "Go to line N.  If N is negative, count from the end of file."
+  (interactive "nGo to line: ")
+  (let ((start-pos vlfi-start-pos)
+        (end-pos vlfi-end-pos)
+        (pos (point))
+        (success nil))
+    (unwind-protect
+        (if (< 0 n)
+            (progn (vlfi-beginning-of-file)
+                   (goto-char (point-min))
+                   (setq success (vlfi-re-search "[\n\C-m]" (1- n)
+                                                 nil 0)))
+          (vlfi-end-of-file)
+          (goto-char (point-max))
+          (setq success (vlfi-re-search "[\n\C-m]" (- n) t 0)))
+      (if success
+          (message "Onto line %s" n)
+        (vlfi-move-to-chunk start-pos end-pos)
+        (goto-char pos)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;; occur
+
+(defvar vlfi-occur-mode-map
+  (let ((map (make-sparse-keymap)))
+    (define-key map "n" 'vlfi-occur-next-match)
+    (define-key map "p" 'vlfi-occur-prev-match)
+    (define-key map "\C-m" 'vlfi-occur-visit)
+    (define-key map [mouse-1] 'vlfi-occur-visit)
+    (define-key map "o" 'vlfi-occur-show)
+    map)
+  "Keymap for command `vlfi-occur-mode'.")
+
+(define-derived-mode vlfi-occur-mode special-mode "VLFI[occur]"
+  "Major mode for showing occur matches of VLFI opened files.")
+
+(defun vlfi-occur-next-match ()
+  "Move cursor to next match."
+  (interactive)
+  (if (eq (get-char-property (point) 'face) 'match)
+      (goto-char (next-single-property-change (point) 'face)))
+  (goto-char (or (text-property-any (point) (point-max) 'face 'match)
+                 (text-property-any (point-min) (point)
+                                    'face 'match))))
+
+(defun vlfi-occur-prev-match ()
+  "Move cursor to previous match."
+  (interactive)
+  (if (eq (get-char-property (point) 'face) 'match)
+      (goto-char (previous-single-property-change (point) 'face)))
+  (while (not (eq (get-char-property (point) 'face) 'match))
+    (goto-char (or (previous-single-property-change (point) 'face)
+                   (point-max)))))
+
+(defun vlfi-occur-show (&optional event)
+  "Visit current `vlfi-occur' link in a vlfi buffer but stay in the \
+occur buffer.  If original VLFI buffer has been killed,
+open new VLFI session each time.
+EVENT may hold details of the invocation."
+  (interactive (list last-nonmenu-event))
+  (let ((occur-buffer (if event
+                          (window-buffer (posn-window
+                                          (event-end event)))
+                        (current-buffer))))
+    (vlfi-occur-visit event)
+    (pop-to-buffer occur-buffer)))
+
+(defun vlfi-occur-visit (&optional event)
+  "Visit current `vlfi-occur' link in a vlfi buffer.
+If original VLFI buffer has been killed,
+open new VLFI session each time.
+EVENT may hold details of the invocation."
+  (interactive (list last-nonmenu-event))
+  (when event
+    (set-buffer (window-buffer (posn-window (event-end event))))
+    (goto-char (posn-point (event-end event))))
+  (let* ((pos (point))
+         (pos-relative (- pos (line-beginning-position) 1))
+         (file (get-char-property pos 'file)))
+    (if file
+        (let ((chunk-start (get-char-property pos 'chunk-start))
+              (chunk-end (get-char-property pos 'chunk-end))
+              (buffer (get-char-property pos 'buffer))
+              (match-pos (+ (get-char-property pos 'line-pos)
+                            pos-relative)))
+          (or (buffer-live-p buffer)
+              (let ((occur-buffer (current-buffer)))
+                (setq buffer (vlfi file))
+                (switch-to-buffer occur-buffer)))
+          (pop-to-buffer buffer)
+          (if (buffer-modified-p)
+              (cond ((and (= vlfi-start-pos chunk-start)
+                          (= vlfi-end-pos chunk-end))
+                     (goto-char match-pos))
+                    ((y-or-n-p "VLFI buffer has been modified.  \
+Really jump to new chunk? ")
+                     (vlfi-move-to-chunk chunk-start chunk-end)
+                     (goto-char match-pos)))
+            (vlfi-move-to-chunk chunk-start chunk-end)
+            (goto-char match-pos))))))
+
+(defun vlfi-occur (regexp)
+  "Make whole file occur style index for REGEXP.
+Prematurely ending indexing will still show what's found so far."
+  (interactive (list (read-regexp "List lines matching regexp"
+                                  (if regexp-history
+                                      (car regexp-history)))))
+  (let ((start-pos vlfi-start-pos)
+        (end-pos vlfi-end-pos)
+        (pos (point)))
+    (vlfi-beginning-of-file)
+    (goto-char (point-min))
+    (unwind-protect (vlfi-build-occur regexp)
+      (vlfi-move-to-chunk start-pos end-pos)
+      (goto-char pos))))
+
+(defun vlfi-build-occur (regexp)
+  "Build occur style index for REGEXP."
+  (let ((line 1)
+        (last-match-line 0)
+        (last-line-pos (point-min))
+        (file buffer-file-name)
+        (total-matches 0)
+        (match-end-pos (+ vlfi-start-pos (position-bytes (point))))
+        (occur-buffer (generate-new-buffer
+                       (concat "*VLFI-occur " (file-name-nondirectory
+                                               buffer-file-name)
+                               "*")))
+        (line-regexp (concat "\\(?5:[\n\C-m]\\)\\|\\(?10:"
+                             regexp "\\)"))
+        (batch-step (/ vlfi-batch-size 8))
+        (end-of-file nil)
+        (reporter (make-progress-reporter
+                   (concat "Building index for " regexp "...")
+                   vlfi-start-pos vlfi-file-size)))
+    (unwind-protect
+        (progn
+          (while (not end-of-file)
+            (if (re-search-forward line-regexp nil t)
+                (progn
+                  (setq match-end-pos (+ vlfi-start-pos
+                                         (position-bytes
+                                          (match-end 0))))
+                  (if (match-string 5)
+                      (setq line (1+ line) ; line detected
+                            last-line-pos (point))
+                    (let* ((chunk-start vlfi-start-pos)
+                           (chunk-end vlfi-end-pos)
+                           (vlfi-buffer (current-buffer))
+                           (line-pos (line-beginning-position))
+                           (line-text (buffer-substring
+                                       line-pos (line-end-position))))
+                      (with-current-buffer occur-buffer
+                        (unless (= line last-match-line) ;new match line
+                          (insert "\n:") ; insert line number
+                          (let* ((overlay-pos (1- (point)))
+                                 (overlay (make-overlay
+                                           overlay-pos
+                                           (1+ overlay-pos))))
+                            (overlay-put overlay 'before-string
+                                         (propertize
+                                          (number-to-string line)
+                                          'face 'shadow)))
+                          (insert (propertize line-text ; insert line
+                                              'file file
+                                              'buffer vlfi-buffer
+                                              'chunk-start chunk-start
+                                              'chunk-end chunk-end
+                                              'mouse-face '(highlight)
+                                              'line-pos line-pos
+                                              'help-echo
+                                              (format "Move to line %d"
+                                                      line))))
+                        (setq last-match-line line
+                              total-matches (1+ total-matches))
+                        (let ((line-start (1+
+                                           (line-beginning-position)))
+                              (match-pos (match-beginning 10)))
+                          (add-text-properties ; mark match
+                           (+ line-start match-pos (- last-line-pos))
+                           (+ line-start (match-end 10)
+                              (- last-line-pos))
+                           (list 'face 'match
+                                 'help-echo
+                                 (format "Move to match %d"
+                                         total-matches))))))))
+              (setq end-of-file (= vlfi-end-pos vlfi-file-size))
+              (unless end-of-file
+                (let ((batch-move (- vlfi-end-pos batch-step)))
+                  (vlfi-move-to-batch (if (< batch-move match-end-pos)
+                                          match-end-pos
+                                        batch-move) t))
+                (goto-char (if (< vlfi-start-pos match-end-pos)
+                               (or (byte-to-position (- match-end-pos
+                                                        vlfi-start-pos))
+                                   (point-min))
+                             (point-min)))
+                (setq last-match-line 0
+                      last-line-pos (line-beginning-position))
+                (progress-reporter-update reporter vlfi-end-pos))))
+          (progress-reporter-done reporter))
+      (if (zerop total-matches)
+          (progn (with-current-buffer occur-buffer
+                   (set-buffer-modified-p nil))
+                 (kill-buffer occur-buffer)
+                 (message "No matches for \"%s\"" regexp))
+        (with-current-buffer occur-buffer
+          (goto-char (point-min))
+          (insert (propertize
+                   (format "%d matches from %d lines for \"%s\" \
+in file: %s" total-matches line regexp file)
+                   'face 'underline))
+          (set-buffer-modified-p nil)
+          (forward-char 2)
+          (vlfi-occur-mode))
+        (display-buffer occur-buffer)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; editing
@@ -427,6 +732,7 @@ Search is performed chunk by chunk in `vlfi-batch-size' memory."
     (set-keymap-parent map text-mode-map)
     (define-key map "\C-c\C-c" 'vlfi-write)
     (define-key map "\C-c\C-q" 'vlfi-discard-edit)
+    (define-key map "\C-v" vlfi-mode-map)
     map)
   "Keymap for command `vlfi-edit-mode'.")
 
@@ -441,6 +747,7 @@ or \\[vlfi-discard-edit] to discard changes.")))
 (defun vlfi-discard-edit ()
   "Discard edit and refresh chunk from file."
   (interactive)
+  (set-buffer-modified-p nil)
   (vlfi-move-to-chunk vlfi-start-pos vlfi-end-pos)
   (vlfi-mode)
   (message "Switched to VLFI mode."))
@@ -452,110 +759,101 @@ or \\[vlfi-discard-edit] to discard changes.")))
   "Write current chunk to file.  Always return true to disable save.
 If changing size of chunk shift remaining file content."
   (interactive)
-  (when (and (derived-mode-p 'vlfi-mode)
-             (buffer-modified-p)
-             (or (verify-visited-file-modtime)
+  (when (and (buffer-modified-p)
+             (or (verify-visited-file-modtime (current-buffer))
                  (y-or-n-p "File has changed since visited or saved.  \
 Save anyway? ")))
-    (let ((size-change (- vlfi-end-pos vlfi-start-pos
+    (let ((pos (point))
+          (size-change (- vlfi-end-pos vlfi-start-pos
                           (length (encode-coding-region
                                    (point-min) (point-max)
-                                   buffer-file-coding-system t))))
-          (pos (point)))
+                                   buffer-file-coding-system t)))))
       (cond ((zerop size-change)
              (write-region nil nil buffer-file-name vlfi-start-pos t))
             ((< 0 size-change)
              (vlfi-file-shift-back size-change))
             (t (vlfi-file-shift-forward (- size-change))))
+      (vlfi-move-to-chunk vlfi-start-pos vlfi-end-pos)
       (goto-char pos))
-    (vlfi-move-to-chunk vlfi-start-pos vlfi-end-pos)
-    (vlfi-mode)
-    t))
+    (vlfi-mode))
+  t)
 
 (defun vlfi-file-shift-back (size-change)
   "Shift file contents SIZE-CHANGE bytes back."
-  (let ((coding-system buffer-file-coding-system))
-    (write-region nil nil buffer-file-name vlfi-start-pos t)
-    (setq buffer-file-coding-system nil)
-    (let ((read-start-pos vlfi-end-pos)
-          (reporter (make-progress-reporter "Adjusting file content"
-                                            vlfi-end-pos
-                                            vlfi-file-size)))
-      (while (vlfi-shift-batch read-start-pos (- read-start-pos
-                                                 size-change))
-        (setq read-start-pos (+ read-start-pos vlfi-batch-size))
-        (progress-reporter-update reporter read-start-pos))
-      ;; pad end with space
-      (erase-buffer)
-      (insert-char 32 size-change)
-      (write-region nil nil buffer-file-name (- vlfi-file-size
-                                                size-change) t)
-      (progress-reporter-done reporter))
-    (setq buffer-file-coding-system coding-system)))
+  (write-region nil nil buffer-file-name vlfi-start-pos t)
+  (buffer-disable-undo)
+  (let ((read-start-pos vlfi-end-pos)
+        (coding-system-for-write 'no-conversion)
+        (reporter (make-progress-reporter "Adjusting file content..."
+                                          vlfi-end-pos
+                                          vlfi-file-size)))
+    (while (vlfi-shift-batch read-start-pos (- read-start-pos
+                                               size-change))
+      (setq read-start-pos (+ read-start-pos vlfi-batch-size))
+      (progress-reporter-update reporter read-start-pos))
+    ;; pad end with space
+    (erase-buffer)
+    (insert-char 32 size-change)
+    (write-region nil nil buffer-file-name (- vlfi-file-size
+                                              size-change) t)
+    (progress-reporter-done reporter)))
 
 (defun vlfi-shift-batch (read-pos write-pos)
   "Read `vlfi-batch-size' bytes from READ-POS and write them \
 back at WRITE-POS.  Return nil if EOF is reached, t otherwise."
   (erase-buffer)
-  (setq vlfi-file-size (nth 7 (file-attributes buffer-file-name)))
+  (or (verify-visited-file-modtime (current-buffer))
+      (setq vlfi-file-size (vlfi-get-file-size buffer-file-name)))
   (let ((read-end (+ read-pos vlfi-batch-size)))
     (insert-file-contents-literally buffer-file-name nil
                                     read-pos
                                     (min vlfi-file-size read-end))
-    (write-region nil nil buffer-file-name write-pos t)
+    (write-region nil nil buffer-file-name write-pos 0)
     (< read-end vlfi-file-size)))
 
 (defun vlfi-file-shift-forward (size-change)
   "Shift file contents SIZE-CHANGE bytes forward.
 Done by saving content up front and then writing previous batch."
-  (let ((vlfi-buffer (current-buffer))
-        (temp-buffer (generate-new-buffer (concat " "
-                                                  (buffer-name))))
-        (coding-system buffer-file-coding-system))
-    (let ((file buffer-file-name))
-      (set-buffer temp-buffer)
-      (setq buffer-file-name file))
-    (set-buffer vlfi-buffer)
-    (let ((read-buffer temp-buffer)
-          (write-buffer vlfi-buffer)
-          (size (+ vlfi-batch-size size-change))
-          (read-pos vlfi-end-pos)
-          (write-pos vlfi-start-pos)
-          swap-buffer
-          (reporter (make-progress-reporter "Adjusting file content"
-                                            vlfi-start-pos
-                                            vlfi-file-size)))
-      (while (vlfi-shift-batches size read-buffer read-pos
-                                 write-buffer write-pos)
-        (setq swap-buffer read-buffer
-              read-buffer write-buffer
-              write-buffer swap-buffer
-              write-pos (+ read-pos size-change)
-              read-pos (+ read-pos size))
-        (progress-reporter-update reporter write-pos))
-      (progress-reporter-done reporter))
-    (kill-buffer temp-buffer)
-    (set-buffer vlfi-buffer)
-    (setq buffer-file-coding-system coding-system)))
+  (buffer-disable-undo)
+  (let ((size (+ vlfi-batch-size size-change))
+        (read-pos vlfi-end-pos)
+        (write-pos vlfi-start-pos)
+        (reporter (make-progress-reporter "Adjusting file content..."
+                                          vlfi-start-pos
+                                          vlfi-file-size)))
+    (when (vlfi-shift-batches size read-pos write-pos t)
+      (setq write-pos (+ read-pos size-change)
+            read-pos (+ read-pos size))
+      (progress-reporter-update reporter write-pos)
+      (let ((coding-system-for-write 'no-conversion))
+        (while (vlfi-shift-batches size read-pos write-pos nil)
+          (setq write-pos (+ read-pos size-change)
+                read-pos (+ read-pos size))
+          (progress-reporter-update reporter write-pos))))
+    (progress-reporter-done reporter)))
 
-(defun vlfi-shift-batches (size read-buffer read-pos
-                                write-buffer write-pos)
-  "Read SIZE bytes in READ-BUFFER starting from READ-POS.
-Then write contents of WRITE-BUFFER to buffer file at WRITE-POS.
+(defun vlfi-shift-batches (size read-pos write-pos hide-read)
+  "Append SIZE bytes of file starting at READ-POS.
+Then write initial buffer content to file at WRITE-POS.
+If HIDE-READ is non nil, temporarily hide literal read content.
 Return nil if EOF is reached, t otherwise."
-  (let* ((file-size (nth 7 (file-attributes buffer-file-name)))
-         (read-more (< read-pos file-size)))
+  (or (verify-visited-file-modtime (current-buffer))
+      (setq vlfi-file-size (vlfi-get-file-size buffer-file-name)))
+  (let ((read-more (< read-pos vlfi-file-size))
+        (start-write-pos (point-min))
+        (end-write-pos (point-max)))
     (when read-more
-      ;; read
-      (set-buffer read-buffer)
-      (erase-buffer)
-      (setq buffer-file-coding-system nil)
+      (goto-char end-write-pos)
       (insert-file-contents-literally buffer-file-name nil read-pos
-                                      (min file-size (+ read-pos
-                                                        size))))
+                                      (min vlfi-file-size (+ read-pos
+                                                             size))))
     ;; write
-    (set-buffer write-buffer)
-    (write-region nil nil buffer-file-name write-pos t)
+    (if hide-read ; hide literal region if user has to choose encoding
+        (narrow-to-region start-write-pos end-write-pos))
+    (write-region start-write-pos end-write-pos
+                  buffer-file-name write-pos 0)
+    (delete-region start-write-pos end-write-pos)
+    (if hide-read (widen))
     read-more))
 
 (provide 'vlfi)