From fc053642333eff13f07a77d328b14a5270383369 Mon Sep 17 00:00:00 2001 From: Pablo Santiago Blum de Aguiar Date: Wed, 4 Nov 2015 23:08:13 -0200 Subject: [PATCH] #398 & #408: Support non-ascii IO in Python 2 --- tests/rules/conftest.py | 2 +- tests/rules/test_dirty_unzip.py | 76 +++++++++++++++++++----------- tests/rules/test_fix_file.py | 20 +++++++- tests/rules/test_grep_recursive.py | 4 ++ tests/test_corrector.py | 4 ++ tests/test_shells.py | 34 +++++++++---- tests/test_types.py | 8 ++++ thefuck/corrector.py | 2 +- thefuck/rules/cd_correction.py | 8 +++- thefuck/rules/dirty_unzip.py | 4 +- thefuck/rules/fix_file.py | 2 +- thefuck/rules/grep_recursive.py | 2 +- thefuck/shells.py | 15 +++--- thefuck/types.py | 3 +- 14 files changed, 129 insertions(+), 55 deletions(-) diff --git a/tests/rules/conftest.py b/tests/rules/conftest.py index 94152a6c..99a5c0d3 100644 --- a/tests/rules/conftest.py +++ b/tests/rules/conftest.py @@ -3,4 +3,4 @@ import pytest @pytest.fixture(autouse=True) def generic_shell(monkeypatch): - monkeypatch.setattr('thefuck.shells.and_', lambda *x: ' && '.join(x)) + monkeypatch.setattr('thefuck.shells.and_', lambda *x: u' && '.join(x)) diff --git a/tests/rules/test_dirty_unzip.py b/tests/rules/test_dirty_unzip.py index 9c0b4e51..44de835f 100644 --- a/tests/rules/test_dirty_unzip.py +++ b/tests/rules/test_dirty_unzip.py @@ -1,50 +1,72 @@ +# -*- coding: utf-8 -*- + import os import pytest import zipfile from thefuck.rules.dirty_unzip import match, get_new_command, side_effect from tests.utils import Command +from unicodedata import normalize @pytest.fixture def zip_error(tmpdir): - path = os.path.join(str(tmpdir), 'foo.zip') + def zip_error_inner(filename): + path = os.path.join(str(tmpdir), filename) - def reset(path): - with zipfile.ZipFile(path, 'w') as archive: - archive.writestr('a', '1') - archive.writestr('b', '2') - archive.writestr('c', '3') + def reset(path): + with zipfile.ZipFile(path, 'w') as archive: + archive.writestr('a', '1') + archive.writestr('b', '2') + archive.writestr('c', '3') - archive.writestr('d/e', '4') + archive.writestr('d/e', '4') - archive.extractall() + archive.extractall() - os.chdir(str(tmpdir)) - reset(path) + os.chdir(str(tmpdir)) + reset(path) - assert set(os.listdir('.')) == {'foo.zip', 'a', 'b', 'c', 'd'} - assert set(os.listdir('./d')) == {'e'} + dir_list = os.listdir(u'.') + if filename not in dir_list: + filename = normalize('NFD', filename) + + assert set(dir_list) == {filename, 'a', 'b', 'c', 'd'} + assert set(os.listdir('./d')) == {'e'} + return zip_error_inner -@pytest.mark.parametrize('script', [ - 'unzip foo', - 'unzip foo.zip']) -def test_match(zip_error, script): +@pytest.mark.parametrize('script,filename', [ + (u'unzip café', u'café.zip'), + (u'unzip café.zip', u'café.zip'), + (u'unzip foo', u'foo.zip'), + (u'unzip foo.zip', u'foo.zip')]) +def test_match(zip_error, script, filename): + zip_error(filename) assert match(Command(script=script)) -@pytest.mark.parametrize('script', [ - 'unzip foo', - 'unzip foo.zip']) -def test_side_effect(zip_error, script): +@pytest.mark.parametrize('script,filename', [ + (u'unzip café', u'café.zip'), + (u'unzip café.zip', u'café.zip'), + (u'unzip foo', u'foo.zip'), + (u'unzip foo.zip', u'foo.zip')]) +def test_side_effect(zip_error, script, filename): + zip_error(filename) side_effect(Command(script=script), None) - assert set(os.listdir('.')) == {'foo.zip', 'd'} + + dir_list = os.listdir(u'.') + if filename not in set(dir_list): + filename = normalize('NFD', filename) + + assert set(dir_list) == {filename, 'd'} -@pytest.mark.parametrize('script,fixed', [ - ('unzip foo', 'unzip foo -d foo'), - (R"unzip foo\ bar.zip", R"unzip foo\ bar.zip -d 'foo bar'"), - (R"unzip 'foo bar.zip'", R"unzip 'foo bar.zip' -d 'foo bar'"), - ('unzip foo.zip', 'unzip foo.zip -d foo')]) -def test_get_new_command(zip_error, script, fixed): +@pytest.mark.parametrize('script,fixed,filename', [ + (u'unzip café', u"unzip café -d 'café'", u'café.zip'), + (u'unzip foo', u'unzip foo -d foo', u'foo.zip'), + (u"unzip foo\\ bar.zip", u"unzip foo\\ bar.zip -d 'foo bar'", u'foo.zip'), + (u"unzip 'foo bar.zip'", u"unzip 'foo bar.zip' -d 'foo bar'", u'foo.zip'), + (u'unzip foo.zip', u'unzip foo.zip -d foo', u'foo.zip')]) +def test_get_new_command(zip_error, script, fixed, filename): + zip_error(filename) assert get_new_command(Command(script=script)) == fixed diff --git a/tests/rules/test_fix_file.py b/tests/rules/test_fix_file.py index c285aa2d..30f5fa1d 100644 --- a/tests/rules/test_fix_file.py +++ b/tests/rules/test_fix_file.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import pytest import os from thefuck.rules.fix_file import match, get_new_command @@ -87,6 +89,20 @@ Traceback (most recent call last): TypeError: first argument must be string or compiled pattern """), +(u'python café.py', u'café.py', 8, None, '', +u""" +Traceback (most recent call last): + File "café.py", line 8, in + match("foo") + File "café.py", line 5, in match + m = re.search(None, command) + File "/usr/lib/python3.4/re.py", line 170, in search + return _compile(pattern, flags).search(string) + File "/usr/lib/python3.4/re.py", line 293, in _compile + raise TypeError("first argument must be string or compiled pattern") +TypeError: first argument must be string or compiled pattern +"""), + ('ruby a.rb', 'a.rb', 3, None, '', """ a.rb:3: syntax error, unexpected keyword_end @@ -227,7 +243,7 @@ def test_get_new_command_with_settings(mocker, monkeypatch, test, settings): if test[3]: assert (get_new_command(cmd) == - 'dummy_editor {} +{}:{} && {}'.format(test[1], test[2], test[3], test[0])) + u'dummy_editor {} +{}:{} && {}'.format(test[1], test[2], test[3], test[0])) else: assert (get_new_command(cmd) == - 'dummy_editor {} +{} && {}'.format(test[1], test[2], test[0])) + u'dummy_editor {} +{} && {}'.format(test[1], test[2], test[0])) diff --git a/tests/rules/test_grep_recursive.py b/tests/rules/test_grep_recursive.py index 4e82688f..1bf248f1 100644 --- a/tests/rules/test_grep_recursive.py +++ b/tests/rules/test_grep_recursive.py @@ -1,11 +1,15 @@ +# -*- coding: utf-8 -*- + from thefuck.rules.grep_recursive import match, get_new_command from tests.utils import Command def test_match(): assert match(Command('grep blah .', stderr='grep: .: Is a directory')) + assert match(Command(u'grep café .', stderr='grep: .: Is a directory')) assert not match(Command()) def test_get_new_command(): assert get_new_command(Command('grep blah .')) == 'grep -r blah .' + assert get_new_command(Command(u'grep café .')) == u'grep -r café .' diff --git a/tests/test_corrector.py b/tests/test_corrector.py index 399d0878..529025b5 100644 --- a/tests/test_corrector.py +++ b/tests/test_corrector.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import pytest from pathlib import PosixPath from thefuck import corrector, conf @@ -53,7 +55,9 @@ def test_organize_commands(): """Ensures that the function removes duplicates and sorts commands.""" commands = [CorrectedCommand('ls'), CorrectedCommand('ls -la', priority=9000), CorrectedCommand('ls -lh', priority=100), + CorrectedCommand(u'echo café', priority=200), CorrectedCommand('ls -lh', priority=9999)] assert list(organize_commands(iter(commands))) \ == [CorrectedCommand('ls'), CorrectedCommand('ls -lh', priority=100), + CorrectedCommand(u'echo café', priority=200), CorrectedCommand('ls -la', priority=9000)] diff --git a/tests/test_shells.py b/tests/test_shells.py index 557b7907..39875fea 100644 --- a/tests/test_shells.py +++ b/tests/test_shells.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import pytest from thefuck import shells @@ -35,6 +37,7 @@ class TestGeneric(object): def test_put_to_history(self, builtins_open, shell): assert shell.put_to_history('ls') is None + assert shell.put_to_history(u'echo café') is None assert builtins_open.call_count == 0 def test_and_(self, shell): @@ -55,6 +58,10 @@ class TestGeneric(object): # so just ignore them: assert list(shell.get_history()) == [] + def test_split_command(self, shell): + assert shell.split_command('ls') == ['ls'] + assert shell.split_command(u'echo café') == [u'echo', u'café'] + @pytest.mark.usefixtures('isfile') class TestBash(object): @@ -83,10 +90,13 @@ class TestBash(object): def test_to_shell(self, shell): assert shell.to_shell('pwd') == 'pwd' - def test_put_to_history(self, builtins_open, shell): - shell.put_to_history('ls') + @pytest.mark.parametrize('entry, entry_utf8', [ + ('ls', 'ls\n'), + (u'echo café', 'echo café\n')]) + def test_put_to_history(self, entry, entry_utf8, builtins_open, shell): + shell.put_to_history(entry) builtins_open.return_value.__enter__.return_value. \ - write.assert_called_once_with('ls\n') + write.assert_called_once_with(entry_utf8) def test_and_(self, shell): assert shell.and_('ls', 'cd') == 'ls && cd' @@ -152,12 +162,15 @@ class TestFish(object): def test_to_shell(self, shell): assert shell.to_shell('pwd') == 'pwd' - def test_put_to_history(self, builtins_open, mocker, shell): + @pytest.mark.parametrize('entry, entry_utf8', [ + ('ls', '- cmd: ls\n when: 1430707243\n'), + (u'echo café', '- cmd: echo café\n when: 1430707243\n')]) + def test_put_to_history(self, entry, entry_utf8, builtins_open, mocker, shell): mocker.patch('thefuck.shells.time', return_value=1430707243.3517463) - shell.put_to_history('ls') + shell.put_to_history(entry) builtins_open.return_value.__enter__.return_value. \ - write.assert_called_once_with('- cmd: ls\n when: 1430707243\n') + write.assert_called_once_with(entry_utf8) def test_and_(self, shell): assert shell.and_('foo', 'bar') == 'foo; and bar' @@ -212,12 +225,15 @@ class TestZsh(object): def test_to_shell(self, shell): assert shell.to_shell('pwd') == 'pwd' - def test_put_to_history(self, builtins_open, mocker, shell): + @pytest.mark.parametrize('entry, entry_utf8', [ + ('ls', ': 1430707243:0;ls\n'), + (u'echo café', ': 1430707243:0;echo café\n')]) + def test_put_to_history(self, entry, entry_utf8, builtins_open, mocker, shell): mocker.patch('thefuck.shells.time', return_value=1430707243.3517463) - shell.put_to_history('ls') + shell.put_to_history(entry) builtins_open.return_value.__enter__.return_value. \ - write.assert_called_once_with(': 1430707243:0;ls\n') + write.assert_called_once_with(entry_utf8) def test_and_(self, shell): assert shell.and_('ls', 'cd') == 'ls && cd' diff --git a/tests/test_types.py b/tests/test_types.py index 54452a0a..c5f16445 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from subprocess import PIPE from mock import Mock from pathlib import Path @@ -19,6 +21,12 @@ class TestCorrectedCommand(object): assert {CorrectedCommand('ls', None, 100), CorrectedCommand('ls', None, 200)} == {CorrectedCommand('ls')} + def test_representable(self): + assert '{}'.format(CorrectedCommand('ls', None, 100)) == \ + 'CorrectedCommand(script=ls, side_effect=None, priority=100)' + assert u'{}'.format(CorrectedCommand(u'echo café', None, 100)) == \ + u'CorrectedCommand(script=echo café, side_effect=None, priority=100)' + class TestRule(object): def test_from_path(self, mocker): diff --git a/thefuck/corrector.py b/thefuck/corrector.py index 2b16cca6..f08c7a74 100644 --- a/thefuck/corrector.py +++ b/thefuck/corrector.py @@ -55,7 +55,7 @@ def organize_commands(corrected_commands): key=lambda corrected_command: corrected_command.priority) logs.debug('Corrected commands: '.format( - ', '.join(str(cmd) for cmd in [first_command] + sorted_commands))) + ', '.join(u'{}'.format(cmd) for cmd in [first_command] + sorted_commands))) for command in sorted_commands: yield command diff --git a/thefuck/rules/cd_correction.py b/thefuck/rules/cd_correction.py index 5adab757..231ca799 100644 --- a/thefuck/rules/cd_correction.py +++ b/thefuck/rules/cd_correction.py @@ -1,6 +1,7 @@ """Attempts to spellcheck and correct failed cd commands""" import os +import six from difflib import get_close_matches from thefuck.specific.sudo import sudo_support from thefuck.rules import cd_mkdir @@ -36,7 +37,10 @@ def get_new_command(command): dest = command.script_parts[1].split(os.sep) if dest[-1] == '': dest = dest[:-1] - cwd = os.getcwd() + if six.PY2: + cwd = os.getcwdu() + else: + cwd = os.getcwd() for directory in dest: if directory == ".": continue @@ -48,7 +52,7 @@ def get_new_command(command): cwd = os.path.join(cwd, best_matches[0]) else: return cd_mkdir.get_new_command(command) - return 'cd "{0}"'.format(cwd) + return u'cd "{0}"'.format(cwd) enabled_by_default = True diff --git a/thefuck/rules/dirty_unzip.py b/thefuck/rules/dirty_unzip.py index f15d6e99..3e45ea37 100644 --- a/thefuck/rules/dirty_unzip.py +++ b/thefuck/rules/dirty_unzip.py @@ -19,7 +19,7 @@ def _zip_file(command): if c.endswith('.zip'): return c else: - return '{}.zip'.format(c) + return u'{}.zip'.format(c) @for_app('unzip') @@ -29,7 +29,7 @@ def match(command): def get_new_command(command): - return '{} -d {}'.format(command.script, quote(_zip_file(command)[:-4])) + return u'{} -d {}'.format(command.script, quote(_zip_file(command)[:-4])) def side_effect(old_cmd, command): diff --git a/thefuck/rules/fix_file.py b/thefuck/rules/fix_file.py index 753426e2..fcae4a67 100644 --- a/thefuck/rules/fix_file.py +++ b/thefuck/rules/fix_file.py @@ -58,7 +58,7 @@ def match(command): return _search(command.stderr) or _search(command.stdout) -@default_settings({'fixlinecmd': '{editor} {file} +{line}', +@default_settings({'fixlinecmd': u'{editor} {file} +{line}', 'fixcolcmd': None}) def get_new_command(command): m = _search(command.stderr) or _search(command.stdout) diff --git a/thefuck/rules/grep_recursive.py b/thefuck/rules/grep_recursive.py index af92300d..3f793a18 100644 --- a/thefuck/rules/grep_recursive.py +++ b/thefuck/rules/grep_recursive.py @@ -7,4 +7,4 @@ def match(command): def get_new_command(command): - return 'grep -r {}'.format(command.script[5:]) + return u'grep -r {}'.format(command.script[5:]) diff --git a/thefuck/shells.py b/thefuck/shells.py index d67414de..10f56a85 100644 --- a/thefuck/shells.py +++ b/thefuck/shells.py @@ -11,7 +11,6 @@ import io import os import shlex import six -import sys from .utils import DEVNULL, memoize, cache @@ -51,10 +50,11 @@ class Generic(object): history_file_name = self._get_history_file_name() if os.path.isfile(history_file_name): with open(history_file_name, 'a') as history: - if sys.version_info >= (3, 0): - history.write(self._get_history_line(command_script)) + entry = self._get_history_line(command_script) + if six.PY2: + history.write(entry.encode('utf-8')) else: - history.write(self._get_history_line(command_script).encode("utf-8")) + history.write(entry) def _script_from_history(self, line): """Returns prepared history line. @@ -84,10 +84,9 @@ class Generic(object): def split_command(self, command): """Split the command using shell-like syntax.""" - if sys.version_info >= (3, 0): - return shlex.split(command) - else: - return shlex.split(command.encode("utf-8")) + if six.PY2: + return [s.decode('utf8') for s in shlex.split(command.encode('utf8'))] + return shlex.split(command) def quote(self, s): """Return a shell-escaped version of the string s.""" diff --git a/thefuck/types.py b/thefuck/types.py index 3bc39030..45542490 100644 --- a/thefuck/types.py +++ b/thefuck/types.py @@ -267,7 +267,7 @@ class CorrectedCommand(object): return (self.script, self.side_effect).__hash__() def __repr__(self): - return 'CorrectedCommand(script={}, side_effect={}, priority={})'.format( + return u'CorrectedCommand(script={}, side_effect={}, priority={})'.format( self.script, self.side_effect, self.priority) def run(self, old_cmd): @@ -279,4 +279,5 @@ class CorrectedCommand(object): if self.side_effect: compatibility_call(self.side_effect, old_cmd, self.script) shells.put_to_history(self.script) + # This depends on correct setting of PYTHONIOENCODING by the alias: print(self.script)