Merge pull request #411 from scorphus/unicode

Support non-ascii content in Python 2

tests/rules/conftest.py

@@ -3,4 +3,4 @@ import pytest
 
 @pytest.fixture(autouse=True)
 def generic_shell(monkeypatch):
-    monkeypatch.setattr('thefuck.shells.and_', lambda *x: ' && '.join(x))
+    monkeypatch.setattr('thefuck.shells.and_', lambda *x: u' && '.join(x))

tests/rules/test_dirty_unzip.py

@@ -1,50 +1,72 @@
+# -*- coding: utf-8 -*-
+
 import os
 import pytest
 import zipfile
 from thefuck.rules.dirty_unzip import match, get_new_command, side_effect
 from tests.utils import Command
+from unicodedata import normalize
 
 
 @pytest.fixture
 def zip_error(tmpdir):
-    path = os.path.join(str(tmpdir), 'foo.zip')
+    def zip_error_inner(filename):
+        path = os.path.join(str(tmpdir), filename)
 
-    def reset(path):
-        with zipfile.ZipFile(path, 'w') as archive:
-            archive.writestr('a', '1')
-            archive.writestr('b', '2')
-            archive.writestr('c', '3')
+        def reset(path):
+            with zipfile.ZipFile(path, 'w') as archive:
+                archive.writestr('a', '1')
+                archive.writestr('b', '2')
+                archive.writestr('c', '3')
 
-            archive.writestr('d/e', '4')
+                archive.writestr('d/e', '4')
 
-            archive.extractall()
+                archive.extractall()
 
-    os.chdir(str(tmpdir))
-    reset(path)
+        os.chdir(str(tmpdir))
+        reset(path)
 
-    assert set(os.listdir('.')) == {'foo.zip', 'a', 'b', 'c', 'd'}
-    assert set(os.listdir('./d')) == {'e'}
+        dir_list = os.listdir(u'.')
+        if filename not in dir_list:
+            filename = normalize('NFD', filename)
+
+        assert set(dir_list) == {filename, 'a', 'b', 'c', 'd'}
+        assert set(os.listdir('./d')) == {'e'}
+    return zip_error_inner
 
 
-@pytest.mark.parametrize('script', [
-    'unzip foo',
-    'unzip foo.zip'])
-def test_match(zip_error, script):
+@pytest.mark.parametrize('script,filename', [
+    (u'unzip café', u'café.zip'),
+    (u'unzip café.zip', u'café.zip'),
+    (u'unzip foo', u'foo.zip'),
+    (u'unzip foo.zip', u'foo.zip')])
+def test_match(zip_error, script, filename):
+    zip_error(filename)
     assert match(Command(script=script))
 
 
-@pytest.mark.parametrize('script', [
-    'unzip foo',
-    'unzip foo.zip'])
-def test_side_effect(zip_error, script):
+@pytest.mark.parametrize('script,filename', [
+    (u'unzip café', u'café.zip'),
+    (u'unzip café.zip', u'café.zip'),
+    (u'unzip foo', u'foo.zip'),
+    (u'unzip foo.zip', u'foo.zip')])
+def test_side_effect(zip_error, script, filename):
+    zip_error(filename)
     side_effect(Command(script=script), None)
-    assert set(os.listdir('.')) == {'foo.zip', 'd'}
+
+    dir_list = os.listdir(u'.')
+    if filename not in set(dir_list):
+        filename = normalize('NFD', filename)
+
+    assert set(dir_list) == {filename, 'd'}
 
 
-@pytest.mark.parametrize('script,fixed', [
-    ('unzip foo', 'unzip foo -d foo'),
-    (R"unzip foo\ bar.zip", R"unzip foo\ bar.zip -d 'foo bar'"),
-    (R"unzip 'foo bar.zip'", R"unzip 'foo bar.zip' -d 'foo bar'"),
-    ('unzip foo.zip', 'unzip foo.zip -d foo')])
-def test_get_new_command(zip_error, script, fixed):
+@pytest.mark.parametrize('script,fixed,filename', [
+    (u'unzip café', u"unzip café -d 'café'", u'café.zip'),
+    (u'unzip foo', u'unzip foo -d foo', u'foo.zip'),
+    (u"unzip foo\\ bar.zip", u"unzip foo\\ bar.zip -d 'foo bar'", u'foo.zip'),
+    (u"unzip 'foo bar.zip'", u"unzip 'foo bar.zip' -d 'foo bar'", u'foo.zip'),
+    (u'unzip foo.zip', u'unzip foo.zip -d foo', u'foo.zip')])
+def test_get_new_command(zip_error, script, fixed, filename):
+    zip_error(filename)
     assert get_new_command(Command(script=script)) == fixed

tests/rules/test_fix_file.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import pytest
 import os
 from thefuck.rules.fix_file import match, get_new_command
@@ -87,6 +89,20 @@ Traceback (most recent call last):
 TypeError: first argument must be string or compiled pattern
 """),
 
+(u'python café.py', u'café.py', 8, None, '',
+u"""
+Traceback (most recent call last):
+  File "café.py", line 8, in <module>
+    match("foo")
+  File "café.py", line 5, in match
+    m = re.search(None, command)
+  File "/usr/lib/python3.4/re.py", line 170, in search
+    return _compile(pattern, flags).search(string)
+  File "/usr/lib/python3.4/re.py", line 293, in _compile
+    raise TypeError("first argument must be string or compiled pattern")
+TypeError: first argument must be string or compiled pattern
+"""),
+
 ('ruby a.rb', 'a.rb', 3, None, '',
 """
 a.rb:3: syntax error, unexpected keyword_end
@@ -227,7 +243,7 @@ def test_get_new_command_with_settings(mocker, monkeypatch, test, settings):
 
     if test[3]:
         assert (get_new_command(cmd) ==
-            'dummy_editor {} +{}:{} && {}'.format(test[1], test[2], test[3], test[0]))
+            u'dummy_editor {} +{}:{} && {}'.format(test[1], test[2], test[3], test[0]))
     else:
         assert (get_new_command(cmd) ==
-            'dummy_editor {} +{} && {}'.format(test[1], test[2], test[0]))
+            u'dummy_editor {} +{} && {}'.format(test[1], test[2], test[0]))

tests/rules/test_grep_recursive.py

@@ -1,12 +1,15 @@
+# -*- coding: utf-8 -*-
+
 from thefuck.rules.grep_recursive import match, get_new_command
 from tests.utils import Command
 
 
 def test_match():
     assert match(Command('grep blah .', stderr='grep: .: Is a directory'))
+    assert match(Command(u'grep café .', stderr='grep: .: Is a directory'))
     assert not match(Command())
 
 
 def test_get_new_command():
-    assert get_new_command(
-        Command('grep blah .')) == 'grep -r blah .'
+    assert get_new_command(Command('grep blah .')) == 'grep -r blah .'
+    assert get_new_command(Command(u'grep café .')) == u'grep -r café .'

tests/test_corrector.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import pytest
 from pathlib import PosixPath
 from thefuck import corrector, conf
@@ -53,7 +55,9 @@ def test_organize_commands():
     """Ensures that the function removes duplicates and sorts commands."""
     commands = [CorrectedCommand('ls'), CorrectedCommand('ls -la', priority=9000),
                 CorrectedCommand('ls -lh', priority=100),
+                CorrectedCommand(u'echo café', priority=200),
                 CorrectedCommand('ls -lh', priority=9999)]
     assert list(organize_commands(iter(commands))) \
         == [CorrectedCommand('ls'), CorrectedCommand('ls -lh', priority=100),
+            CorrectedCommand(u'echo café', priority=200),
             CorrectedCommand('ls -la', priority=9000)]

tests/test_shells.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import pytest
 from thefuck import shells
 
@@ -35,6 +37,7 @@ class TestGeneric(object):
 
     def test_put_to_history(self, builtins_open, shell):
         assert shell.put_to_history('ls') is None
+        assert shell.put_to_history(u'echo café') is None
         assert builtins_open.call_count == 0
 
     def test_and_(self, shell):
@@ -48,6 +51,7 @@ class TestGeneric(object):
         assert 'alias FUCK' in shell.app_alias('FUCK')
         assert 'thefuck' in shell.app_alias('fuck')
         assert 'TF_ALIAS' in shell.app_alias('fuck')
+        assert 'PYTHONIOENCODING=utf-8' in shell.app_alias('fuck')
 
     def test_get_history(self, history_lines, shell):
         history_lines(['ls', 'rm'])
@@ -55,6 +59,10 @@ class TestGeneric(object):
         # so just ignore them:
         assert list(shell.get_history()) == []
 
+    def test_split_command(self, shell):
+        assert shell.split_command('ls') == ['ls']
+        assert shell.split_command(u'echo café') == [u'echo', u'café']
+
 
 @pytest.mark.usefixtures('isfile')
 class TestBash(object):
@@ -83,10 +91,13 @@ class TestBash(object):
     def test_to_shell(self, shell):
         assert shell.to_shell('pwd') == 'pwd'
 
-    def test_put_to_history(self, builtins_open, shell):
-        shell.put_to_history('ls')
+    @pytest.mark.parametrize('entry, entry_utf8', [
+        ('ls', 'ls\n'),
+        (u'echo café', 'echo café\n')])
+    def test_put_to_history(self, entry, entry_utf8, builtins_open, shell):
+        shell.put_to_history(entry)
         builtins_open.return_value.__enter__.return_value. \
-            write.assert_called_once_with('ls\n')
+            write.assert_called_once_with(entry_utf8)
 
     def test_and_(self, shell):
         assert shell.and_('ls', 'cd') == 'ls && cd'
@@ -102,6 +113,7 @@ class TestBash(object):
         assert 'alias FUCK' in shell.app_alias('FUCK')
         assert 'thefuck' in shell.app_alias('fuck')
         assert 'TF_ALIAS' in shell.app_alias('fuck')
+        assert 'PYTHONIOENCODING=utf-8' in shell.app_alias('fuck')
 
     def test_get_history(self, history_lines, shell):
         history_lines(['ls', 'rm'])
@@ -152,12 +164,15 @@ class TestFish(object):
     def test_to_shell(self, shell):
         assert shell.to_shell('pwd') == 'pwd'
 
-    def test_put_to_history(self, builtins_open, mocker, shell):
+    @pytest.mark.parametrize('entry, entry_utf8', [
+        ('ls', '- cmd: ls\n   when: 1430707243\n'),
+        (u'echo café', '- cmd: echo café\n   when: 1430707243\n')])
+    def test_put_to_history(self, entry, entry_utf8, builtins_open, mocker, shell):
         mocker.patch('thefuck.shells.time',
                      return_value=1430707243.3517463)
-        shell.put_to_history('ls')
+        shell.put_to_history(entry)
         builtins_open.return_value.__enter__.return_value. \
-            write.assert_called_once_with('- cmd: ls\n   when: 1430707243\n')
+            write.assert_called_once_with(entry_utf8)
 
     def test_and_(self, shell):
         assert shell.and_('foo', 'bar') == 'foo; and bar'
@@ -179,6 +194,12 @@ class TestFish(object):
         assert 'function FUCK' in shell.app_alias('FUCK')
         assert 'thefuck' in shell.app_alias('fuck')
         assert 'TF_ALIAS' in shell.app_alias('fuck')
+        assert 'PYTHONIOENCODING=utf-8' in shell.app_alias('fuck')
+
+    def test_get_history(self, history_lines, shell):
+        history_lines(['- cmd: ls', '  when: 1432613911',
+                       '- cmd: rm', '  when: 1432613916'])
+        assert list(shell.get_history()) == ['ls', 'rm']
 
 
 @pytest.mark.usefixtures('isfile')
@@ -207,12 +228,15 @@ class TestZsh(object):
     def test_to_shell(self, shell):
         assert shell.to_shell('pwd') == 'pwd'
 
-    def test_put_to_history(self, builtins_open, mocker, shell):
+    @pytest.mark.parametrize('entry, entry_utf8', [
+        ('ls', ': 1430707243:0;ls\n'),
+        (u'echo café', ': 1430707243:0;echo café\n')])
+    def test_put_to_history(self, entry, entry_utf8, builtins_open, mocker, shell):
         mocker.patch('thefuck.shells.time',
                      return_value=1430707243.3517463)
-        shell.put_to_history('ls')
+        shell.put_to_history(entry)
         builtins_open.return_value.__enter__.return_value. \
-            write.assert_called_once_with(': 1430707243:0;ls\n')
+            write.assert_called_once_with(entry_utf8)
 
     def test_and_(self, shell):
         assert shell.and_('ls', 'cd') == 'ls && cd'
@@ -229,6 +253,7 @@ class TestZsh(object):
         assert 'alias FUCK' in shell.app_alias('FUCK')
         assert 'thefuck' in shell.app_alias('fuck')
         assert 'TF_ALIAS' in shell.app_alias('fuck')
+        assert 'PYTHONIOENCODING=utf-8' in shell.app_alias('fuck')
 
     def test_get_history(self, history_lines, shell):
         history_lines([': 1432613911:0;ls', ': 1432613916:0;rm'])

tests/test_types.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from subprocess import PIPE
 from mock import Mock
 from pathlib import Path
@@ -19,6 +21,12 @@ class TestCorrectedCommand(object):
         assert {CorrectedCommand('ls', None, 100),
                 CorrectedCommand('ls', None, 200)} == {CorrectedCommand('ls')}
 
+    def test_representable(self):
+        assert '{}'.format(CorrectedCommand('ls', None, 100)) == \
+               'CorrectedCommand(script=ls, side_effect=None, priority=100)'
+        assert u'{}'.format(CorrectedCommand(u'echo café', None, 100)) == \
+               u'CorrectedCommand(script=echo café, side_effect=None, priority=100)'
+
 
 class TestRule(object):
     def test_from_path(self, mocker):