From 16ad9182f1facd5e871b12ff413dc26d5b24f727 Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Fri, 3 Nov 2017 17:24:33 -0400 Subject: [PATCH 1/7] Add en-AU (Australia) to the list of valid languages --- qutebrowser/config/configdata.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index a99152e88..389d41957 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1066,6 +1066,7 @@ spellcheck.languages: - da-DK: Danish (Denmark) - de-DE: German (Germany) - el-GR: Greek (Greece) + - en-AU: English (Australia) - en-CA: English (Canada) - en-GB: English (United Kingdom) - en-US: English (United States) From 2dc0115c8129eddcba117c8f9f868986b34b0c9c Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Fri, 3 Nov 2017 19:20:31 -0400 Subject: [PATCH 2/7] Load the newest version of the dictionary. --- qutebrowser/browser/webengine/spell.py | 20 +++++++++++++++++--- tests/unit/browser/webengine/test_spell.py | 9 +++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/webengine/spell.py b/qutebrowser/browser/webengine/spell.py index a915b1d52..994982315 100644 --- a/qutebrowser/browser/webengine/spell.py +++ b/qutebrowser/browser/webengine/spell.py @@ -21,10 +21,22 @@ import glob import os +import re +from qutebrowser.utils import log from PyQt5.QtCore import QLibraryInfo +def version(filename): + """Extract the version number from the dictionary file name.""" + version_re = re.compile(r""" + .+(?P[0-9]+-[0-9]+)\.bdic + """, re.VERBOSE) + match = version_re.match(filename) + assert match is not None, 'the given dictionary file name is malformed' + return [int(n) for n in match.group('version').split('-')] + + def dictionary_dir(): """Return the path (str) to the QtWebEngine's dictionaries directory.""" datapath = QLibraryInfo.location(QLibraryInfo.DataPath) @@ -32,14 +44,16 @@ def dictionary_dir(): def installed_file(code): - """Return the installed dictionary for the given code. + """Return the newest installed dictionary for the given code. - Return the filename of the installed dictionary or None - if the dictionary is not installed. + Return the filename of the installed dictionary with the highest version + number or None if the dictionary is not installed. """ pathname = os.path.join(dictionary_dir(), '{}*.bdic'.format(code)) matching_dicts = glob.glob(pathname) if matching_dicts: + log.config.debug('Found files for dict {}: {}'.format(code, matching_dicts)) + matching_dicts = sorted(matching_dicts, key=version) with_extension = os.path.basename(matching_dicts[0]) return os.path.splitext(with_extension)[0] else: diff --git a/tests/unit/browser/webengine/test_spell.py b/tests/unit/browser/webengine/test_spell.py index 2aebeffd5..39f6e8601 100644 --- a/tests/unit/browser/webengine/test_spell.py +++ b/tests/unit/browser/webengine/test_spell.py @@ -21,6 +21,11 @@ from qutebrowser.browser.webengine import spell +def test_version(): + assert spell.version('en-US-8-0.bdic') == [8, 0] + assert spell.version('pl-PL-3-0.bdic') == [3, 0] + + def test_installed_file_dictionary_does_not_exist(tmpdir, monkeypatch): monkeypatch.setattr( spell, 'dictionary_dir', lambda: '/some-non-existing-dir') @@ -34,7 +39,7 @@ def test_installed_file_dictionary_not_installed(tmpdir, monkeypatch): def test_installed_file_dictionary_installed(tmpdir, monkeypatch): monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) - for lang_file in ['en-US-7-1.bdic', 'pl-PL-3-0.bdic']: + for lang_file in ['en-US-11-0.bdic', 'en-US-7-1.bdic', 'pl-PL-3-0.bdic']: (tmpdir / lang_file).ensure() - assert spell.installed_file('en-US') == 'en-US-7-1' + assert spell.installed_file('en-US') == 'en-US-11-0' assert spell.installed_file('pl-PL') == 'pl-PL-3-0' From 3ac2cfdf73326e8731ba212fcd8492ab4a99b7f0 Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Sat, 4 Nov 2017 18:16:05 -0400 Subject: [PATCH 3/7] Support updating dictionaries and removing old versions. --- qutebrowser/browser/webengine/spell.py | 30 +- .../browser/webengine/webenginesettings.py | 8 +- scripts/dev/update_3rdparty.py | 6 +- scripts/dictcli.py | 282 ++++++++++++++++++ scripts/install_dict.py | 189 ------------ tests/end2end/conftest.py | 2 +- tests/unit/browser/webengine/test_spell.py | 14 +- tests/unit/scripts/test_dictcli.py | 159 ++++++++++ tests/unit/scripts/test_install_dict.py | 71 ----- 9 files changed, 474 insertions(+), 287 deletions(-) create mode 100755 scripts/dictcli.py delete mode 100755 scripts/install_dict.py create mode 100644 tests/unit/scripts/test_dictcli.py delete mode 100644 tests/unit/scripts/test_install_dict.py diff --git a/qutebrowser/browser/webengine/spell.py b/qutebrowser/browser/webengine/spell.py index 994982315..cb7da541c 100644 --- a/qutebrowser/browser/webengine/spell.py +++ b/qutebrowser/browser/webengine/spell.py @@ -30,10 +30,11 @@ from PyQt5.QtCore import QLibraryInfo def version(filename): """Extract the version number from the dictionary file name.""" version_re = re.compile(r""" - .+(?P[0-9]+-[0-9]+)\.bdic + .+-(?P[0-9]+-[0-9]+?)\.bdic """, re.VERBOSE) match = version_re.match(filename) - assert match is not None, 'the given dictionary file name is malformed' + assert match is not None, \ + 'the given dictionary file name is malformed: {}'.format(filename) return [int(n) for n in match.group('version').split('-')] @@ -43,18 +44,23 @@ def dictionary_dir(): return os.path.join(datapath, 'qtwebengine_dictionaries') -def installed_file(code): +def local_files(code): + """Return all installed dictionaries for the given code.""" + pathname = os.path.join(dictionary_dir(), '{}*.bdic'.format(code)) + matching_dicts = glob.glob(pathname) + files = [] + for matching_dict in sorted(matching_dicts, key=version, reverse=True): + filename = os.path.basename(matching_dict) + log.config.debug('Found file for dict {}: {}'.format(code, filename)) + files.append(filename) + return files + + +def local_filename(code): """Return the newest installed dictionary for the given code. Return the filename of the installed dictionary with the highest version number or None if the dictionary is not installed. """ - pathname = os.path.join(dictionary_dir(), '{}*.bdic'.format(code)) - matching_dicts = glob.glob(pathname) - if matching_dicts: - log.config.debug('Found files for dict {}: {}'.format(code, matching_dicts)) - matching_dicts = sorted(matching_dicts, key=version) - with_extension = os.path.basename(matching_dicts[0]) - return os.path.splitext(with_extension)[0] - else: - return None + all_installed = local_files(code) + return os.path.splitext(all_installed[0])[0] if all_installed else None diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 12503a7c0..7f1d50e78 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -134,12 +134,12 @@ class DictionaryLanguageSetter(DefaultProfileSetter): super().__init__('setSpellCheckLanguages', default=[]) def _find_installed(self, code): - installed_file = spell.installed_file(code) - if not installed_file: + local_filename = spell.local_filename(code) + if not local_filename: message.warning( - "Language {} is not installed - see scripts/install_dict.py " + "Language {} is not installed - see scripts/dict.py " "in qutebrowser's sources".format(code)) - return installed_file + return local_filename def _set(self, value, settings=None): if settings is not None: diff --git a/scripts/dev/update_3rdparty.py b/scripts/dev/update_3rdparty.py index a72e9368e..9c67db107 100755 --- a/scripts/dev/update_3rdparty.py +++ b/scripts/dev/update_3rdparty.py @@ -31,7 +31,7 @@ import sys sys.path.insert( 0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -from scripts import install_dict +from scripts import dictcli from qutebrowser.config import configdata @@ -119,9 +119,9 @@ def update_ace(): def test_dicts(): """Test available dictionaries.""" configdata.init() - for lang in install_dict.available_languages(): + for lang in dictcli.available_languages(): print('Testing dictionary {}... '.format(lang.code), end='') - lang_url = urllib.parse.urljoin(install_dict.API_URL, lang.file_path) + lang_url = urllib.parse.urljoin(dictcli.API_URL, lang.remote_path) request = urllib.request.Request(lang_url, method='HEAD') response = urllib.request.urlopen(request) if response.status == 200: diff --git a/scripts/dictcli.py b/scripts/dictcli.py new file mode 100755 index 000000000..961d5d7a3 --- /dev/null +++ b/scripts/dictcli.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Michal Siedlaczek + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""A script installing Hunspell dictionaries. + +Use: python -m scripts.dictcli [-h] {list,update,remove-old,install} ... +""" + +import argparse +import base64 +import json +import os +import sys +import re +import urllib.request + +import attr + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) +from qutebrowser.browser.webengine import spell +from qutebrowser.config import configdata + + +API_URL = 'https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries.git/+/master/' + + +class InvalidLanguageError(Exception): + + """Raised when requesting invalid languages.""" + + def __init__(self, invalid_langs): + msg = 'invalid languages: {}'.format(', '.join(invalid_langs)) + super().__init__(msg) + + +@attr.s +class Language: + + """Dictionary language specs.""" + + code = attr.ib() + name = attr.ib() + remote_filename = attr.ib() + local_filename = attr.ib(default=None) + _file_extension = attr.ib('bdic', init=False) + + def __attrs_post_init__(self): + if self.local_filename is None: + self.local_filename = spell.local_filename(self.code) + + @property + def remote_path(self): + """Resolve the filename with extension the remote dictionary.""" + return '.'.join([self.remote_filename, self._file_extension]) + + @property + def local_path(self): + """Resolve the filename with extension the local dictionary.""" + if self.local_filename is None: + return None + return '.'.join([self.local_filename, self._file_extension]) + + @property + def remote_version(self): + """Resolve the version of the local dictionary.""" + return spell.version(self.remote_path) + + @property + def local_version(self): + """Resolve the version of the local dictionary.""" + local_path = self.local_path + if local_path is None: + return None + return spell.version(local_path) + + +def get_argparser(): + """Get the argparse parser.""" + desc = 'Install and manage Hunspell dictionaries for QtWebEngine.' + parser = argparse.ArgumentParser(prog='dictcli', + description=desc) + subparsers = parser.add_subparsers(help='Command', dest='cmd') + subparsers.add_parser('list', + help='Display the list of available languages.') + subparsers.add_parser('update', + help='Update dictionaries') + subparsers.add_parser('remove-old', + help='Remove old versions of dictionaries.') + + install_parser = subparsers.add_parser('install', + help='Install dictionaries') + install_parser.add_argument('language', + nargs='*', help="A list of languages to install.") + + return parser + + +def version_str(version): + return '.'.join(str(n) for n in version) + + +def print_list(languages): + """Print the list of available languages.""" + pat = '{:<7}{:<26}{:<8}{:<5}' + print(pat.format('Code', 'Name', 'Version', 'Installed')) + for lang in languages: + remote_version = version_str(lang.remote_version) + local_version = '-' + if lang.local_version is not None: + local_version = version_str(lang.local_version) + if lang.local_version < lang.remote_version: + local_version += ' - update available!' + print(pat.format(lang.code, lang.name, remote_version, local_version)) + + +def valid_languages(): + """Return a mapping from valid language codes to their names.""" + option = configdata.DATA['spellcheck.languages'] + return option.typ.valtype.valid_values.descriptions + + +def parse_entry(entry): + """Parse an entry from the remote API.""" + dict_re = re.compile(r""" + (?P(?P[a-z]{2}(-[A-Z]{2})?).*)\.bdic + """, re.VERBOSE) + match = dict_re.match(entry['name']) + if match is not None: + return match.group('code'), match.group('filename') + else: + return None + + +def language_list_from_api(): + """Return a JSON with a list of available languages from Google API.""" + listurl = API_URL + '?format=JSON' + response = urllib.request.urlopen(listurl) + # A special 5-byte prefix must be stripped from the response content + # See: https://github.com/google/gitiles/issues/22 + # https://github.com/google/gitiles/issues/82 + json_content = response.read()[5:] + entries = json.loads(json_content.decode('utf-8'))['entries'] + parsed_entries = [parse_entry(entry) for entry in entries] + return [entry for entry in parsed_entries if entry is not None] + + +def latest_yet(code2file, code, filename): + """Determine wether the latest version so far.""" + if code not in code2file: + return True + return spell.version(code2file[code]) < spell.version(filename) + + +def available_languages(): + """Return a list of Language objects of all available languages.""" + lang_map = valid_languages() + api_list = language_list_from_api() + code2file = {} + for code, filename in api_list: + if latest_yet(code2file, code, filename): + code2file[code] = filename + print(code2file) + return [ + Language(code, name, code2file[code]) + for code, name in lang_map.items() + if code in code2file + ] + + +def download_dictionary(url, dest): + """Download a decoded dictionary file.""" + response = urllib.request.urlopen(url) + decoded = base64.decodebytes(response.read()) + with open(dest, 'bw') as dict_file: + dict_file.write(decoded) + + +def filter_languages(languages, selected): + """Filter a list of languages based on an inclusion list. + + Args: + languages: a list of languages to filter + selected: a list of keys to select + """ + filtered_languages = [] + for language in languages: + if language.code in selected: + filtered_languages.append(language) + selected.remove(language.code) + if selected: + raise InvalidLanguageError(selected) + return filtered_languages + + +def install_lang(lang): + """Install a single lang given by the argument.""" + lang_url = API_URL + lang.remote_path + '?format=TEXT' + if not os.path.isdir(spell.dictionary_dir()): + msg = '{} does not exist, creating the directory' + print(msg.format(spell.dictionary_dir())) + os.makedirs(spell.dictionary_dir()) + print('Downloading {}'.format(lang_url)) + dest = os.path.join(spell.dictionary_dir(), lang.remote_path) + download_dictionary(lang_url, dest) + print('Done.') + + +def install(languages): + """Install languages.""" + for lang in languages: + try: + print('Installing {}: {}'.format(lang.code, lang.name)) + install_lang(lang) + except PermissionError as e: + print(e) + sys.exit(1) + + +def update(languages): + installed = [lang for lang in languages if lang.local_version is not None] + for lang in installed: + if lang.local_version < lang.remote_version: + print('Upgrading {} from {} to {}'.format( + lang.code, + version_str(lang.local_version), + version_str(lang.remote_version))) + install_lang(lang) + + +def remove_old(languages): + installed = [lang for lang in languages if lang.local_version is not None] + for lang in installed: + local_files = spell.local_files(lang.code) + for old_file in local_files[1:]: + os.remove(os.path.join(spell.dictionary_dir(), old_file)) + + +def main(): + if configdata.DATA is None: + configdata.init() + parser = get_argparser() + argv = sys.argv[1:] + args = parser.parse_args(argv) + languages = available_languages() + if args.cmd is None: + parser.print_usage() + exit(1) + elif args.cmd == 'list': + print_list(languages) + elif args.cmd == 'update': + update(languages) + elif args.cmd == 'remove-old': + remove_old(languages) + elif not args.language: + print('You must provide a list of languages to install.') + exit(1) + else: + try: + install(filter_languages(languages, args.language)) + except InvalidLanguageError as e: + print(e) + + +if __name__ == '__main__': + main() diff --git a/scripts/install_dict.py b/scripts/install_dict.py deleted file mode 100755 index 79c9ff122..000000000 --- a/scripts/install_dict.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2017 Michal Siedlaczek - -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""A script installing Hunspell dictionaries. - -Use: python -m scripts.install_dict [--list] [lang [lang [...]]] -""" - -import argparse -import base64 -import json -import os -import sys -import re -import urllib.request - -import attr - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) -from qutebrowser.browser.webengine import spell -from qutebrowser.config import configdata - - -API_URL = 'https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries.git/+/master/' - - -class InvalidLanguageError(Exception): - - """Raised when requesting invalid languages.""" - - def __init__(self, invalid_langs): - msg = 'invalid languages: {}'.format(', '.join(invalid_langs)) - super().__init__(msg) - - -@attr.s -class Language: - - """Dictionary language specs.""" - - code = attr.ib(None) - name = attr.ib(None) - file_basename = attr.ib(None) - file_extension = attr.ib('bdic') - - @property - def file_path(self): - return '.'.join([self.file_basename, self.file_extension]) - - -def get_argparser(): - """Get the argparse parser.""" - desc = 'Install Hunspell dictionaries for QtWebEngine.' - parser = argparse.ArgumentParser(prog='install_dict', - description=desc) - parser.add_argument('-l', '--list', action='store_true', - help="Display the list of available languages.") - parser.add_argument('languages', nargs='*', - help="A list of languages to install.") - return parser - - -def print_list(languages): - for lang in languages: - print(lang.code, lang.name, sep='\t') - - -def valid_languages(): - """Return a mapping from valid language codes to their names.""" - option = configdata.DATA['spellcheck.languages'] - return option.typ.valtype.valid_values.descriptions - - -def language_list_from_api(): - """Return a JSON with a list of available languages from Google API.""" - listurl = API_URL + '?format=JSON' - response = urllib.request.urlopen(listurl) - # A special 5-byte prefix must be stripped from the response content - # See: https://github.com/google/gitiles/issues/22 - # https://github.com/google/gitiles/issues/82 - json_content = response.read()[5:] - entries = json.loads(json_content.decode('utf-8'))['entries'] - return entries - - -def available_languages(): - """Return a list of Language objects of all available languages.""" - lang_map = valid_languages() - api_list = language_list_from_api() - dict_re = re.compile(r""" - (?P(?P[a-z]{2}(-[A-Z]{2})?).*)\.bdic - """, re.VERBOSE) - code2file = {} - for lang in api_list: - match = dict_re.match(lang['name']) - if match is not None: - code2file[match.group('dict')] = match.group('filename') - return [ - Language(code, name, code2file[code]) - for code, name in lang_map.items() - ] - - -def download_dictionary(url, dest): - """Download a decoded dictionary file.""" - response = urllib.request.urlopen(url) - decoded = base64.decodebytes(response.read()) - with open(dest, 'bw') as dict_file: - dict_file.write(decoded) - - -def filter_languages(languages, selected): - """Filter a list of languages based on an inclusion list. - - Args: - languages: a list of languages to filter - selected: a list of keys to select - """ - filtered_languages = [] - for language in languages: - if language.code in selected: - filtered_languages.append(language) - selected.remove(language.code) - if selected: - raise InvalidLanguageError(selected) - return filtered_languages - - -def install_lang(lang): - """Install a single lang given by the argument.""" - print('Installing {}: {}'.format(lang.code, lang.name)) - lang_url = API_URL + lang.file_path + '?format=TEXT' - if not os.path.isdir(spell.dictionary_dir()): - warn_msg = '{} does not exist, creating the directory' - print(warn_msg.format(spell.dictionary_dir())) - os.makedirs(spell.dictionary_dir()) - print('Downloading {}'.format(lang_url)) - dest = os.path.join(spell.dictionary_dir(), lang.file_path) - download_dictionary(lang_url, dest) - print('Done.') - - -def install(languages): - """Install languages.""" - for lang in languages: - try: - install_lang(lang) - except PermissionError as e: - print(e) - sys.exit(1) - - -def main(): - if configdata.DATA is None: - configdata.init() - parser = get_argparser() - argv = sys.argv[1:] - args = parser.parse_args(argv) - languages = available_languages() - if args.list: - print_list(languages) - elif not args.languages: - parser.print_usage() - else: - try: - install(filter_languages(languages, args.languages)) - except InvalidLanguageError as e: - print(e) - - -if __name__ == '__main__': - main() diff --git a/tests/end2end/conftest.py b/tests/end2end/conftest.py index c236f819a..b2f7b1484 100644 --- a/tests/end2end/conftest.py +++ b/tests/end2end/conftest.py @@ -131,7 +131,7 @@ def _get_dictionary_tag(tag): event = match.group('event') dictionary = match.group('dict') - has_dict = spell.installed_file(dictionary) is not None + has_dict = spell.local_filename(dictionary) is not None if event == 'must_have_dict': return pytest.mark.skipif(not has_dict, reason=tag) elif event == 'cannot_have_dict': diff --git a/tests/unit/browser/webengine/test_spell.py b/tests/unit/browser/webengine/test_spell.py index 39f6e8601..e7944e995 100644 --- a/tests/unit/browser/webengine/test_spell.py +++ b/tests/unit/browser/webengine/test_spell.py @@ -26,20 +26,20 @@ def test_version(): assert spell.version('pl-PL-3-0.bdic') == [3, 0] -def test_installed_file_dictionary_does_not_exist(tmpdir, monkeypatch): +def test_local_filename_dictionary_does_not_exist(tmpdir, monkeypatch): monkeypatch.setattr( spell, 'dictionary_dir', lambda: '/some-non-existing-dir') - assert not spell.installed_file('en-US') + assert not spell.local_filename('en-US') -def test_installed_file_dictionary_not_installed(tmpdir, monkeypatch): +def test_local_filename_dictionary_not_installed(tmpdir, monkeypatch): monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) - assert not spell.installed_file('en-US') + assert not spell.local_filename('en-US') -def test_installed_file_dictionary_installed(tmpdir, monkeypatch): +def test_local_filename_dictionary_installed(tmpdir, monkeypatch): monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) for lang_file in ['en-US-11-0.bdic', 'en-US-7-1.bdic', 'pl-PL-3-0.bdic']: (tmpdir / lang_file).ensure() - assert spell.installed_file('en-US') == 'en-US-11-0' - assert spell.installed_file('pl-PL') == 'pl-PL-3-0' + assert spell.local_filename('en-US') == 'en-US-11-0' + assert spell.local_filename('pl-PL') == 'pl-PL-3-0' diff --git a/tests/unit/scripts/test_dictcli.py b/tests/unit/scripts/test_dictcli.py new file mode 100644 index 000000000..9a3712fcf --- /dev/null +++ b/tests/unit/scripts/test_dictcli.py @@ -0,0 +1,159 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Michal Siedlaczek + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + + +import py.path # pylint: disable=no-name-in-module +import pytest + +from qutebrowser.browser.webengine import spell +from scripts import dictcli +from qutebrowser.config import configdata + + +def afrikaans(): + return dictcli.Language( + 'af-ZA', + 'Afrikaans (South Africa)', + 'af-ZA-3-0') + + +def english(): + return dictcli.Language( + 'en-US', + 'English (United States)', + 'en-US-7-1') + + +def polish(): + return dictcli.Language( + 'pl-PL', + 'Polish (Poland)', + 'pl-PL-3-0') + + +def langs(): + return [afrikaans(), english(), polish()] + + +@pytest.fixture(autouse=True) +def configdata_init(): + """Initialize configdata if needed.""" + if configdata.DATA is None: + configdata.init() + + +def test_language(tmpdir, monkeypatch): + monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) + (tmpdir / 'pl-PL-2-0.bdic').ensure() + assert english().local_filename is None + assert english().local_path is None + assert polish() + + +def test_parse_entry(): + assert dictcli.parse_entry({'name': 'en-US-7-1.bdic'}) == \ + ('en-US', 'en-US-7-1') + + +def test_latest_yet(): + code2file = {'en-US': 'en-US-7-1.bdic'} + assert not dictcli.latest_yet(code2file, 'en-US', 'en-US-7-0.bdic') + assert not dictcli.latest_yet(code2file, 'en-US', 'en-US-7-1.bdic') + assert dictcli.latest_yet(code2file, 'en-US', 'en-US-8-0.bdic') + + +def test_available_languages(tmpdir, monkeypatch): + monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) + for f in ['pl-PL-2-0.bdic', english().remote_path]: + (tmpdir / f).ensure() + monkeypatch.setattr(dictcli, 'language_list_from_api', lambda: [ + (lang.code, lang.remote_filename) for lang in langs() + ]) + assert dictcli.available_languages() == [ + dictcli.Language( + 'af-ZA', 'Afrikaans (South Africa)', + 'af-ZA-3-0', None), + dictcli.Language( + 'en-US', 'English (United States)', + 'en-US-7-1', 'en-US-7-1'), + dictcli.Language( + 'pl-PL', 'Polish (Poland)', + 'pl-PL-3-0', 'pl-PL-2-0') + ] + + +def test_filter_languages(): + filtered_langs = dictcli.filter_languages(langs(), ['af-ZA']) + assert filtered_langs == [afrikaans()] + + filtered_langs = dictcli.filter_languages(langs(), ['pl-PL', 'en-US']) + assert filtered_langs == [english(), polish()] + + with pytest.raises(dictcli.InvalidLanguageError): + dictcli.filter_languages(langs(), ['pl-PL', 'en-GB']) + + +def test_install(tmpdir, monkeypatch): + # given + monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) + monkeypatch.setattr( + dictcli, 'download_dictionary', + lambda _url, dest: py.path.local(dest).ensure()) # pylint: disable=no-member + + # when + dictcli.install(langs()) + + # then + installed_files = [f.basename for f in tmpdir.listdir()] + expected_files = [lang.remote_path for lang in langs()] + assert sorted(installed_files) == sorted(expected_files) + + +def test_update(tmpdir, monkeypatch): + # given + monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) + monkeypatch.setattr( + dictcli, 'download_dictionary', + lambda _url, dest: py.path.local(dest).ensure()) # pylint: disable=no-member + (tmpdir / 'pl-PL-2-0.bdic').ensure() + assert polish().local_version < polish().remote_version + + # when + dictcli.update(langs()) + + # then + assert polish().local_version == polish().remote_version + + +def test_remove_old(tmpdir, monkeypatch): + # given + monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) + monkeypatch.setattr( + dictcli, 'download_dictionary', + lambda _url, dest: py.path.local(dest).ensure()) # pylint: disable=no-member + for f in ['pl-PL-2-0.bdic', polish().remote_path, english().remote_path]: + (tmpdir / f).ensure() + + # when + dictcli.remove_old(langs()) + + # then + installed_files = [f.basename for f in tmpdir.listdir()] + expected_files = [polish().remote_path, english().remote_path] + assert sorted(installed_files) == sorted(expected_files) diff --git a/tests/unit/scripts/test_install_dict.py b/tests/unit/scripts/test_install_dict.py deleted file mode 100644 index 4493d9e9a..000000000 --- a/tests/unit/scripts/test_install_dict.py +++ /dev/null @@ -1,71 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2017 Michal Siedlaczek - -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - - -import py.path # pylint: disable=no-name-in-module -import pytest - -from qutebrowser.browser.webengine import spell -from scripts import install_dict -from qutebrowser.config import configdata - -AFRIKAANS = install_dict.Language( - 'af-ZA', - 'Afrikaans (South Africa)', - 'af-ZA-3-0') -ENGLISH = install_dict.Language( - 'en-US', - 'English (United States)', - 'en-US-7-1') -POLISH = install_dict.Language( - 'pl-PL', - 'Polish (Poland)', - 'pl-PL-3-0') - -LANGUAGE_LIST = [AFRIKAANS, ENGLISH, POLISH] - - -@pytest.fixture(autouse=True) -def configdata_init(): - """Initialize configdata if needed.""" - if configdata.DATA is None: - configdata.init() - - -def test_filter_languages(): - filtered_langs = install_dict.filter_languages(LANGUAGE_LIST, ['af-ZA']) - assert filtered_langs == [AFRIKAANS] - - filtered_langs = install_dict.filter_languages( - LANGUAGE_LIST, ['pl-PL', 'en-US']) - assert filtered_langs == [ENGLISH, POLISH] - - with pytest.raises(install_dict.InvalidLanguageError): - install_dict.filter_languages(LANGUAGE_LIST, ['pl-PL', 'en-GB']) - - -def test_install(tmpdir, monkeypatch): - monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir)) - monkeypatch.setattr( - install_dict, 'download_dictionary', - lambda _url, dest: py.path.local(dest).ensure()) # pylint: disable=no-member - install_dict.install(LANGUAGE_LIST) - installed_files = [f.basename for f in tmpdir.listdir()] - expected_files = [lang.file_path for lang in LANGUAGE_LIST] - assert sorted(installed_files) == sorted(expected_files) From 9ea986a5694fabcf3e2604f067a207daf7f11398 Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Sat, 4 Nov 2017 20:02:49 -0400 Subject: [PATCH 4/7] Fixed typo --- scripts/dictcli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/dictcli.py b/scripts/dictcli.py index 961d5d7a3..d4f04f0ca 100755 --- a/scripts/dictcli.py +++ b/scripts/dictcli.py @@ -162,7 +162,7 @@ def language_list_from_api(): def latest_yet(code2file, code, filename): - """Determine wether the latest version so far.""" + """Determine whether the latest version so far.""" if code not in code2file: return True return spell.version(code2file[code]) < spell.version(filename) @@ -176,7 +176,6 @@ def available_languages(): for code, filename in api_list: if latest_yet(code2file, code, filename): code2file[code] = filename - print(code2file) return [ Language(code, name, code2file[code]) for code, name in lang_map.items() From 51a61cf02d75face86fa9973aa6f649ab1c49d1e Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Sat, 4 Nov 2017 20:03:53 -0400 Subject: [PATCH 5/7] Fix test: sort when comparing file collection --- tests/unit/scripts/test_dictcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/scripts/test_dictcli.py b/tests/unit/scripts/test_dictcli.py index 9a3712fcf..e5408ef68 100644 --- a/tests/unit/scripts/test_dictcli.py +++ b/tests/unit/scripts/test_dictcli.py @@ -85,7 +85,7 @@ def test_available_languages(tmpdir, monkeypatch): monkeypatch.setattr(dictcli, 'language_list_from_api', lambda: [ (lang.code, lang.remote_filename) for lang in langs() ]) - assert dictcli.available_languages() == [ + assert sorted(dictcli.available_languages()) == [ dictcli.Language( 'af-ZA', 'Afrikaans (South Africa)', 'af-ZA-3-0', None), From 855d0312b5c486c0a987c8eb66f9d1a945de0775 Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Sun, 5 Nov 2017 18:12:15 -0500 Subject: [PATCH 6/7] Review fixes --- qutebrowser/browser/webengine/spell.py | 13 ++++++------- qutebrowser/browser/webengine/webenginesettings.py | 2 +- scripts/dictcli.py | 12 ++++-------- tests/unit/browser/webengine/test_spell.py | 4 ++-- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/qutebrowser/browser/webengine/spell.py b/qutebrowser/browser/webengine/spell.py index cb7da541c..ee2fb7813 100644 --- a/qutebrowser/browser/webengine/spell.py +++ b/qutebrowser/browser/webengine/spell.py @@ -23,19 +23,18 @@ import glob import os import re -from qutebrowser.utils import log from PyQt5.QtCore import QLibraryInfo +from qutebrowser.utils import log def version(filename): """Extract the version number from the dictionary file name.""" - version_re = re.compile(r""" - .+-(?P[0-9]+-[0-9]+?)\.bdic - """, re.VERBOSE) + version_re = re.compile(r".+-(?P[0-9]+-[0-9]+?)\.bdic") match = version_re.match(filename) - assert match is not None, \ - 'the given dictionary file name is malformed: {}'.format(filename) - return [int(n) for n in match.group('version').split('-')] + if match is None: + raise ValueError('the given dictionary file name is malformed: {}' + .format(filename)) + return tuple(int(n) for n in match.group('version').split('-')) def dictionary_dir(): diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 7f1d50e78..5ea065cb6 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -137,7 +137,7 @@ class DictionaryLanguageSetter(DefaultProfileSetter): local_filename = spell.local_filename(code) if not local_filename: message.warning( - "Language {} is not installed - see scripts/dict.py " + "Language {} is not installed - see scripts/dictcli.py " "in qutebrowser's sources".format(code)) return local_filename diff --git a/scripts/dictcli.py b/scripts/dictcli.py index d4f04f0ca..2742de8ad 100755 --- a/scripts/dictcli.py +++ b/scripts/dictcli.py @@ -97,6 +97,7 @@ def get_argparser(): parser = argparse.ArgumentParser(prog='dictcli', description=desc) subparsers = parser.add_subparsers(help='Command', dest='cmd') + subparsers.required = True subparsers.add_parser('list', help='Display the list of available languages.') subparsers.add_parser('update', @@ -228,8 +229,7 @@ def install(languages): print('Installing {}: {}'.format(lang.code, lang.name)) install_lang(lang) except PermissionError as e: - print(e) - sys.exit(1) + sys.exit(str(e)) def update(languages): @@ -258,18 +258,14 @@ def main(): argv = sys.argv[1:] args = parser.parse_args(argv) languages = available_languages() - if args.cmd is None: - parser.print_usage() - exit(1) - elif args.cmd == 'list': + if args.cmd == 'list': print_list(languages) elif args.cmd == 'update': update(languages) elif args.cmd == 'remove-old': remove_old(languages) elif not args.language: - print('You must provide a list of languages to install.') - exit(1) + sys.exit('You must provide a list of languages to install.') else: try: install(filter_languages(languages, args.language)) diff --git a/tests/unit/browser/webengine/test_spell.py b/tests/unit/browser/webengine/test_spell.py index e7944e995..3b3a45d7d 100644 --- a/tests/unit/browser/webengine/test_spell.py +++ b/tests/unit/browser/webengine/test_spell.py @@ -22,8 +22,8 @@ from qutebrowser.browser.webengine import spell def test_version(): - assert spell.version('en-US-8-0.bdic') == [8, 0] - assert spell.version('pl-PL-3-0.bdic') == [3, 0] + assert spell.version('en-US-8-0.bdic') == (8, 0) + assert spell.version('pl-PL-3-0.bdic') == (3, 0) def test_local_filename_dictionary_does_not_exist(tmpdir, monkeypatch): From 9153bf8c1928132d256a8fb1af70fbb1aecfc1f5 Mon Sep 17 00:00:00 2001 From: Michal Siedlaczek Date: Sun, 5 Nov 2017 20:12:29 -0500 Subject: [PATCH 7/7] Additional version() test --- tests/unit/browser/webengine/test_spell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/browser/webengine/test_spell.py b/tests/unit/browser/webengine/test_spell.py index 3b3a45d7d..9e38c1883 100644 --- a/tests/unit/browser/webengine/test_spell.py +++ b/tests/unit/browser/webengine/test_spell.py @@ -17,13 +17,15 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . - +import pytest from qutebrowser.browser.webengine import spell def test_version(): assert spell.version('en-US-8-0.bdic') == (8, 0) assert spell.version('pl-PL-3-0.bdic') == (3, 0) + with pytest.raises(ValueError): + spell.version('malformed_filename') def test_local_filename_dictionary_does_not_exist(tmpdir, monkeypatch):