Add support for UTF-16LE and UTF-16BE

src/inputfile.rs

@@ -1,6 +1,8 @@
 use std::fs::File;
 use std::io::{self, BufRead, BufReader};
 
+use content_inspector::{self, ContentType};
+
 use errors::*;
 
 const THEME_PREVIEW_FILE: &[u8] = include_bytes!("../assets/theme_preview.rs");
@@ -8,6 +10,7 @@ const THEME_PREVIEW_FILE: &[u8] = include_bytes!("../assets/theme_preview.rs");
 pub struct InputFileReader<'a> {
     inner: Box<dyn BufRead + 'a>,
     pub first_line: Vec<u8>,
+    pub content_type: ContentType,
 }
 
 impl<'a> InputFileReader<'a> {
@@ -15,18 +18,31 @@ impl<'a> InputFileReader<'a> {
         let mut first_line = vec![];
         reader.read_until(b'\n', &mut first_line).ok();
 
+        let content_type = content_inspector::inspect(&first_line[..]);
+
+        if content_type == ContentType::UTF_16LE {
+            reader.read_until(0x00, &mut first_line).ok();
+        }
+
         InputFileReader {
             inner: Box::new(reader),
             first_line,
+            content_type,
         }
     }
 
     pub fn read_line(&mut self, buf: &mut Vec<u8>) -> io::Result<bool> {
         if self.first_line.is_empty() {
-            self.inner.read_until(b'\n', buf).map(|size| size > 0)
+            let res = self.inner.read_until(b'\n', buf).map(|size| size > 0)?;
+
+            if self.content_type == ContentType::UTF_16LE {
+                self.inner.read_until(0x00, buf).ok();
+            }
+
+            Ok(res)
         } else {
             buf.append(&mut self.first_line);
-            return Ok(true);
+            Ok(true)
         }
     }
 }
@@ -84,3 +100,32 @@ fn basic() {
     assert_eq!(false, res.unwrap());
     assert!(buffer.is_empty());
 }
+
+#[test]
+fn utf16le() {
+    let content = b"\xFF\xFE\x73\x00\x0A\x00\x64\x00";
+    let mut reader = InputFileReader::new(&content[..]);
+
+    assert_eq!(b"\xFF\xFE\x73\x00\x0A\x00", &reader.first_line[..]);
+
+    let mut buffer = vec![];
+
+    let res = reader.read_line(&mut buffer);
+    assert!(res.is_ok());
+    assert_eq!(true, res.unwrap());
+    assert_eq!(b"\xFF\xFE\x73\x00\x0A\x00", &buffer[..]);
+
+    buffer.clear();
+
+    let res = reader.read_line(&mut buffer);
+    assert!(res.is_ok());
+    assert_eq!(true, res.unwrap());
+    assert_eq!(b"\x64\x00", &buffer[..]);
+
+    buffer.clear();
+
+    let res = reader.read_line(&mut buffer);
+    assert!(res.is_ok());
+    assert_eq!(false, res.unwrap());
+    assert!(buffer.is_empty());
+}

src/main.rs

@@ -15,6 +15,7 @@ extern crate atty;
 extern crate console;
 extern crate content_inspector;
 extern crate directories;
+extern crate encoding;
 extern crate git2;
 extern crate syntect;
 extern crate wild;

src/printer.rs

@@ -9,7 +9,10 @@ use console::AnsiCodeIterator;
 use syntect::easy::HighlightLines;
 use syntect::highlighting::Theme;
 
-use content_inspector::{self, ContentType};
+use content_inspector::ContentType;
+
+use encoding::all::{UTF_16BE, UTF_16LE};
+use encoding::{DecoderTrap, Encoding};
 
 use app::Config;
 use assets::HighlightingAssets;
@@ -121,12 +124,9 @@ impl<'a> InteractivePrinter<'a> {
             panel_width = 0;
         }
 
-        // Determine file content type
-        let content_type = content_inspector::inspect(&reader.first_line[..]);
-
         let mut line_changes = None;
 
-        let highlighter = if content_type.is_binary() {
+        let highlighter = if reader.content_type.is_binary() {
             None
         } else {
             // Get the Git modifications
@@ -149,8 +149,8 @@ impl<'a> InteractivePrinter<'a> {
             colors,
             config,
             decorations,
+            content_type: reader.content_type,
             ansi_prefix_sgr: String::new(),
-            content_type,
             line_changes,
             highlighter,
         }
@@ -207,10 +207,11 @@ impl<'a> Printer for InteractivePrinter<'a> {
             _ => ("", "STDIN"),
         };
 
-        let mode = if self.content_type.is_binary() {
-            "   <BINARY>"
-        } else {
-            ""
+        let mode = match self.content_type {
+            ContentType::BINARY => "   <BINARY>",
+            ContentType::UTF_16LE => "   <UTF-16LE>",
+            ContentType::UTF_16BE => "   <UTF-16BE>",
+            _ => ""
         };
 
         writeln!(
@@ -247,7 +248,18 @@ impl<'a> Printer for InteractivePrinter<'a> {
         line_number: usize,
         line_buffer: &[u8],
     ) -> Result<()> {
-        let line = String::from_utf8_lossy(&line_buffer).to_string();
+        let line = match self.content_type {
+            ContentType::BINARY => {
+                return Ok(());
+            }
+            ContentType::UTF_16LE => UTF_16LE
+                .decode(&line_buffer, DecoderTrap::Strict)
+                .unwrap_or("Invalid UTF-16LE".into()),
+            ContentType::UTF_16BE => UTF_16BE
+                .decode(&line_buffer, DecoderTrap::Strict)
+                .unwrap_or("Invalid UTF-16BE".into()),
+            _ => String::from_utf8_lossy(&line_buffer).to_string(),
+        };
         let regions = {
             let highlighter = match self.highlighter {
                 Some(ref mut highlighter) => highlighter,