Merge remote-tracking branch 'origin/pr/2891'

doc/faq.asciidoc

@@ -125,9 +125,9 @@ When using quickmark, you can give them all names, like
 without having to remember the exact website title or address.
 
 How do I use spell checking?::
-    Qutebrowser's support for spell checking is somewhat limited at the moment
+	Configuring spell checking in Qutebrowser depends on the backend in use
-    (see https://github.com/qutebrowser/qutebrowser/issues/700[#700]), but it
+    (see https://github.com/qutebrowser/qutebrowser/issues/700[#700] for
-    can be done.
+	a more detailed discussion).
 +
 For QtWebKit:
 
@@ -145,12 +145,11 @@ For QtWebKit:
 +
 For QtWebEngine:
 
-. Not yet supported unfortunately :-( +
+. Make sure your version of PyQt is 5.8 or higher.
-  Adding it shouldn't be too hard though, since QtWebEngine 5.8 added an API for
+. Use `install_dict.py` script to install dictionaries.
-  this (see
+  Run the script with `-h` for the parameter description.
-  https://github.com/qutebrowser/qutebrowser/issues/700#issuecomment-290780706[this
+. Set `spellcheck.languages` to the desired list of languages, e.g.:
-  comment for a basic example]), so what are you waiting for and why aren't you
+  `:set spellcheck.languages "['en-US', 'pl-PL']"`
-  hacking qutebrowser yet?
 
 How do I use Tor with qutebrowser?::
     Start tor on your machine, and do `:set network proxy socks://localhost:9050/`

qutebrowser/browser/webengine/spell.py

@@ -0,0 +1,46 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Michal Siedlaczek <michal.siedlaczek@gmail.com>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Installing and configuring spell-checking for QtWebEngine."""
+
+import glob
+import os
+
+from PyQt5.QtCore import QLibraryInfo
+
+
+def dictionary_dir():
+    """Return the path (str) to the QtWebEngine's dictionaries directory."""
+    datapath = QLibraryInfo.location(QLibraryInfo.DataPath)
+    return os.path.join(datapath, 'qtwebengine_dictionaries')
+
+
+def installed_file(code):
+    """Return the installed dictionary for the given code.
+
+    Return the filename of the installed dictionary or None
+    if the dictionary is not installed.
+    """
+    pathname = os.path.join(dictionary_dir(), '{}*.bdic'.format(code))
+    matching_dicts = glob.glob(pathname)
+    if matching_dicts:
+        with_extension = os.path.basename(matching_dicts[0])
+        return os.path.splitext(with_extension)[0]
+    else:
+        return None

qutebrowser/browser/webengine/webenginesettings.py

@@ -36,9 +36,9 @@ from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
                                       QWebEngineScript)
 
 from qutebrowser.browser import shared
+from qutebrowser.browser.webengine import spell
 from qutebrowser.config import config, websettings
-from qutebrowser.utils import utils, standarddir, javascript, qtutils
+from qutebrowser.utils import utils, standarddir, javascript, qtutils, message
-
 
 # The default QWebEngineProfile
 default_profile = None
@@ -127,6 +127,29 @@ class PersistentCookiePolicy(DefaultProfileSetter):
         )
 
 
+class DictionaryLanguageSetter(DefaultProfileSetter):
+
+    """Sets paths to dictionary files based on language codes."""
+
+    def __init__(self):
+        super().__init__('setSpellCheckLanguages', default=[])
+
+    def _find_installed(self, code):
+        installed_file = spell.installed_file(code)
+        if not installed_file:
+            message.warning(
+                'Language {} is not installed - see scripts/install_dict.py '
+                'in qutebrowser\'s sources'.format(code))
+        return installed_file
+
+    def _set(self, value, settings=None):
+        if settings is not None:
+            raise ValueError("'settings' may not be set with "
+                             "DictionaryLanguageSetter!")
+        filenames = [self._find_installed(code) for code in value]
+        super()._set([f for f in filenames if f], settings)
+
+
 def _init_stylesheet(profile):
     """Initialize custom stylesheets.
 
@@ -198,6 +221,10 @@ def _init_profiles():
     _init_stylesheet(private_profile)
     _set_http_headers(private_profile)
 
+    if qtutils.version_check('5.8'):
+        default_profile.setSpellCheckEnabled(True)
+        private_profile.setSpellCheckEnabled(True)
+
 
 def init(args):
     """Initialize the global QWebSettings."""
@@ -309,6 +336,10 @@ except AttributeError:
     pass
 
 
+if qtutils.version_check('5.8'):
+    MAPPINGS['spellcheck.languages'] = DictionaryLanguageSetter()
+
+
 if qtutils.version_check('5.9'):
     # https://bugreports.qt.io/browse/QTBUG-58650
     MAPPINGS['content.cookies.store'] = PersistentCookiePolicy()

qutebrowser/config/configdata.yml

@@ -992,6 +992,67 @@ scrolling.smooth:
 
     Note smooth scrolling does not work with the `:scroll-px` command.
 
+## spellcheck
+
+spellcheck.languages:
+  type:
+    name: List
+    valtype:
+      name: String
+      valid_values:
+        - af-ZA: Afrikaans (South Africa)
+        - bg-BG: Bulgarian (Bulgaria)
+        - ca-ES: Catalan (Spain)
+        - cs-CZ: Czech (Czech Republic)
+        - da-DK: Danish (Denmark)
+        - de-DE: German (Germany)
+        - el-GR: Greek (Greece)
+        - en-CA: English (Canada)
+        - en-GB: English (United Kingdom)
+        - en-US: English (United States)
+        - es-ES: Spanish (Spain)
+        - et-EE: Estonian (Estonia)
+        - fa-IR: Farsi (Iran)
+        - fo-FO: Faroese (Faroe Islands)
+        - fr-FR: French (France)
+        - he-IL: Hebrew (Israel)
+        - hi-IN: Hindi (India)
+        - hr-HR: Croatian (Croatia)
+        - hu-HU: Hungarian (Hungary)
+        - id-ID: Indonesian (Indonesia)
+        - it-IT: Italian (Italy)
+        - ko: Korean
+        - lt-LT: Lithuanian (Lithuania)
+        - lv-LV: Latvian (Latvia)
+        - nb-NO: Norwegian (Norway)
+        - nl-NL: Dutch (Netherlands)
+        - pl-PL: Polish (Poland)
+        - pt-BR: Portuguese (Brazil)
+        - pt-PT: Portuguese (Portugal)
+        - ro-RO: Romanian (Romania)
+        - ru-RU: Russian (Russia)
+        - sh: Serbo-Croatian
+        - sk-SK: Slovak (Slovakia)
+        - sl-SI: Slovenian (Slovenia)
+        - sq: Albanian
+        - sr: Serbian
+        - sv-SE: Swedish (Sweden)
+        - ta-IN: Tamil (India)
+        - tg-TG: Tajik (Tajikistan)
+        - tr-TR: Turkish (Turkey)
+        - uk-UA: Ukrainian (Ukraine)
+        - vi-VN: Vietnamese (Viet Nam)
+    none_ok: true
+  default: []
+  desc: >-
+    Spell checking languages.
+
+    You can check for available languages and install dictionaries using
+    scripts/install_dict.py. Run the script with -h/--help for instructions.
+  backend:
+    QtWebKit: false
+    QtWebEngine: Qt 5.8
+
 ## statusbar
 
 statusbar.hide:

scripts/dev/check_coverage.py

@@ -176,6 +176,9 @@ PERFECT_FILES = [
     ('tests/unit/completion/test_listcategory.py',
         'completion/models/listcategory.py'),
 
+    ('tests/unit/browser/webengine/test_spell.py',
+        'browser/webengine/spell.py'),
+
 ]
 
 

scripts/dev/update_3rdparty.py

@@ -27,6 +27,12 @@ import urllib.error
 import shutil
 import json
 import os
+import sys
+
+sys.path.insert(
+    0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+from scripts import install_dict
+from qutebrowser.config import configdata
 
 
 def get_latest_pdfjs_url():
@@ -110,7 +116,22 @@ def update_ace():
     urllib.request.urlcleanup()
 
 
-def run(ace=False, pdfjs=True, fancy_dmg=False, pdfjs_version=None):
+def test_dicts():
+    """Test available dictionaries."""
+    configdata.init()
+    for lang in install_dict.available_languages():
+        print('Testing dictionary {}... '.format(lang.code), end='')
+        lang_url = urllib.parse.urljoin(install_dict.API_URL, lang.file_path)
+        request = urllib.request.Request(lang_url, method='HEAD')
+        response = urllib.request.urlopen(request)
+        if response.status == 200:
+            print('OK')
+        else:
+            print('ERROR: {}'.format(response.status))
+
+
+def run(ace=False, pdfjs=True, fancy_dmg=False, pdfjs_version=None,
+        dicts=False):
     """Update components based on the given arguments."""
     if pdfjs:
         update_pdfjs(pdfjs_version)
@@ -118,6 +139,8 @@ def run(ace=False, pdfjs=True, fancy_dmg=False, pdfjs_version=None):
         update_ace()
     if fancy_dmg:
         update_dmg_makefile()
+    if dicts:
+        test_dicts()
 
 
 def main():
@@ -129,9 +152,14 @@ def main():
         required=False, metavar='VERSION')
     parser.add_argument('--fancy-dmg', help="Update fancy-dmg Makefile",
                         action='store_true')
+    parser.add_argument(
+        '--dicts', '-d',
+        help='Test whether all available dictionaries '
+        'can be reached at the remote repository.',
+        required=False, action='store_true')
     args = parser.parse_args()
     run(ace=True, pdfjs=True, fancy_dmg=args.fancy_dmg,
-        pdfjs_version=args.pdfjs)
+        pdfjs_version=args.pdfjs, dicts=args.dicts)
 
 
 if __name__ == '__main__':

scripts/install_dict.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Michal Siedlaczek <michal.siedlaczek@gmail.com>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A script installing Hunspell dictionaries.
+
+Use: python -m scripts.install_dict [--list] [lang [lang [...]]]
+"""
+
+import argparse
+import base64
+import json
+import os
+import sys
+import re
+import urllib.parse
+import urllib.request
+
+import attr
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
+from qutebrowser.browser.webengine import spell
+from qutebrowser.config import configdata
+
+
+API_URL = 'https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries.git/+/master/'
+
+
+class InvalidLanguageError(Exception):
+
+    """Raised when requesting invalid languages."""
+
+    def __init__(self, invalid_langs):
+        msg = 'invalid languages: {}'.format(', '.join(invalid_langs))
+        super().__init__(msg)
+
+
+@attr.s
+class Language:
+
+    """Dictionary language specs."""
+
+    code = attr.ib(None)
+    name = attr.ib(None)
+    file_basename = attr.ib(None)
+    file_extension = attr.ib('bdic')
+
+    @property
+    def file_path(self):
+        return '.'.join([self.file_basename, self.file_extension])
+
+
+def get_argparser():
+    """Get the argparse parser."""
+    desc = 'Install Hunspell dictionaries for QtWebEngine.'
+    parser = argparse.ArgumentParser(prog='install_dict',
+                                     description=desc)
+    parser.add_argument('-l', '--list', action='store_true',
+                        help="Display the list of available languages.")
+    parser.add_argument('languages', nargs='*',
+                        help="A list of languages to install.")
+    return parser
+
+
+def print_list(languages):
+    for lang in languages:
+        print(lang.code, lang.name, sep='\t')
+
+
+def valid_languages():
+    """Return a mapping from valid language codes to their names."""
+    option = configdata.DATA['spellcheck.languages']
+    return option.typ.valtype.valid_values.descriptions
+
+
+def language_list_from_api():
+    """Return a JSON with a list of available languages from Google API."""
+    listurl = urllib.parse.urljoin(API_URL, '?format=JSON')
+    response = urllib.request.urlopen(listurl)
+    # A special 5-byte prefix must be stripped from the response content
+    # See: https://github.com/google/gitiles/issues/22
+    #      https://github.com/google/gitiles/issues/82
+    json_content = response.read()[5:]
+    entries = json.loads(json_content.decode('utf-8'))['entries']
+    return entries
+
+
+def available_languages():
+    """Return a list of Language objects of all available languages."""
+    lang_map = valid_languages()
+    api_list = language_list_from_api()
+    dict_re = re.compile(r"""
+        (?P<filename>(?P<dict>[a-z]{2}(-[A-Z]{2})?).*)\.bdic
+    """, re.VERBOSE)
+    code2file = {}
+    for lang in api_list:
+        match = dict_re.match(lang['name'])
+        if match is not None:
+            code2file[match.group('dict')] = match.group('filename')
+    return [
+        Language(code, name, code2file[code])
+        for code, name in lang_map.items()
+    ]
+
+
+def download_dictionary(url, dest):
+    """Download a decoded dictionary file."""
+    response = urllib.request.urlopen(url)
+    decoded = base64.decodebytes(response.read())
+    with open(dest, 'bw') as dict_file:
+        dict_file.write(decoded)
+
+
+def filter_languages(languages, selected):
+    """Filter a list of languages based on an inclusion list.
+
+    Args:
+        languages: a list of languages to filter
+        selected: a list of keys to select
+    """
+    filtered_languages = []
+    for language in languages:
+        if language.code in selected:
+            filtered_languages.append(language)
+            selected.remove(language.code)
+    if selected:
+        raise InvalidLanguageError(selected)
+    return filtered_languages
+
+
+def install_lang(lang):
+    """Install a single lang given by the argument."""
+    print('Installing {}: {}'.format(lang.code, lang.name))
+    lang_url = urllib.parse.urljoin(API_URL, lang.file_path, '?format=TEXT')
+    if not os.path.isdir(spell.dictionary_dir()):
+        warn_msg = '{} does not exist, creating the directory'
+        print(warn_msg.format(spell.dictionary_dir()))
+        os.makedirs(spell.dictionary_dir())
+    print('Downloading {}'.format(lang_url))
+    dest = os.path.join(spell.dictionary_dir(), lang.file_path)
+    download_dictionary(lang_url, dest)
+    print('Done.')
+
+
+def install(languages):
+    """Install languages."""
+    for lang in languages:
+        try:
+            install_lang(lang)
+        except PermissionError as e:
+            print(e)
+            sys.exit(1)
+
+
+def main():
+    if configdata.DATA is None:
+        configdata.init()
+    parser = get_argparser()
+    argv = sys.argv[1:]
+    args = parser.parse_args(argv)
+    languages = available_languages()
+    if args.list:
+        print_list(languages)
+    elif not args.languages:
+        parser.print_usage()
+    else:
+        try:
+            install(filter_languages(languages, args.languages))
+        except InvalidLanguageError as e:
+            print(e)
+
+
+if __name__ == '__main__':
+    main()

tests/end2end/conftest.py

@@ -29,6 +29,8 @@ import pstats
 import os.path
 import operator
 
+from qutebrowser.browser.webengine import spell
+
 import pytest
 from PyQt5.QtCore import PYQT_VERSION
 
@@ -116,6 +118,27 @@ def _get_backend_tag(tag):
     return pytest_marks[name](desc)
 
 
+def _get_dictionary_tag(tag):
+    """Handle tags like must_have_dict=en-US for BDD tests."""
+    dict_re = re.compile(r"""
+        (?P<event>must_have_dict|cannot_have_dict)=(?P<dict>[a-z]{2}-[A-Z]{2})
+    """, re.VERBOSE)
+
+    match = dict_re.match(tag)
+    if not match:
+        return None
+
+    event = match.group('event')
+    dictionary = match.group('dict')
+    has_dict = spell.installed_file(dictionary) is not None
+    if event == 'must_have_dict':
+        return pytest.mark.skipif(not has_dict, reason=tag)
+    elif event == 'cannot_have_dict':
+        return pytest.mark.skipif(has_dict, reason=tag)
+    else:
+        return None
+
+
 if not getattr(sys, 'frozen', False):
     def pytest_bdd_apply_tag(tag, function):
         """Handle custom tags for BDD tests.
@@ -123,7 +146,7 @@ if not getattr(sys, 'frozen', False):
         This tries various functions, and if none knows how to handle this tag,
         it returns None so it falls back to pytest-bdd's implementation.
         """
-        funcs = [_get_version_tag, _get_backend_tag]
+        funcs = [_get_version_tag, _get_backend_tag, _get_dictionary_tag]
         for func in funcs:
             mark = func(tag)
             if mark is not None:

tests/end2end/features/misc.feature

@@ -541,3 +541,15 @@ Feature: Various utility commands.
         When I set up "simple" as block lists
         And I run :adblock-update
         Then the message "adblock: Read 1 hosts from 1 sources." should be shown
+
+    ## Spellcheck
+
+    @qtwebkit_skip @qt>=5.8 @cannot_have_dict=af-ZA
+    Scenario: Set valid but not installed language
+        When I run :set spellcheck.languages ['af-ZA']
+        Then the warning "Language af-ZA is not installed *" should be shown
+
+    @qtwebkit_skip @qt>=5.8 @must_have_dict=en-US
+    Scenario: Set valid and installed language
+        When I run :set spellcheck.languages ["en-US"]
+        Then the option spellcheck.languages should be set to ["en-US"]

tests/unit/browser/webengine/test_spell.py

@@ -0,0 +1,40 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Michal Siedlaczek <michal.siedlaczek@gmail.com>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from qutebrowser.browser.webengine import spell
+
+
+def test_installed_file_dictionary_does_not_exist(tmpdir, monkeypatch):
+    monkeypatch.setattr(
+        spell, 'dictionary_dir', lambda: '/some-non-existing-dir')
+    assert not spell.installed_file('en-US')
+
+
+def test_installed_file_dictionary_not_installed(tmpdir, monkeypatch):
+    monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir))
+    assert not spell.installed_file('en-US')
+
+
+def test_installed_file_dictionary_installed(tmpdir, monkeypatch):
+    monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir))
+    for lang_file in ['en-US-7-1.bdic', 'pl-PL-3-0.bdic']:
+        (tmpdir / lang_file).ensure()
+    assert spell.installed_file('en-US') == 'en-US-7-1'
+    assert spell.installed_file('pl-PL') == 'pl-PL-3-0'

tests/unit/scripts/test_install_dict.py

@@ -0,0 +1,71 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Michal Siedlaczek <michal.siedlaczek@gmail.com>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import py.path  # pylint: disable=no-name-in-module
+import pytest
+
+from qutebrowser.browser.webengine import spell
+from scripts import install_dict
+from qutebrowser.config import configdata
+
+AFRIKAANS = install_dict.Language(
+    'af-ZA',
+    'Afrikaans (South Africa)',
+    'af-ZA-3-0')
+ENGLISH = install_dict.Language(
+    'en-US',
+    'English (United States)',
+    'en-US-7-1')
+POLISH = install_dict.Language(
+    'pl-PL',
+    'Polish (Poland)',
+    'pl-PL-3-0')
+
+LANGUAGE_LIST = [AFRIKAANS, ENGLISH, POLISH]
+
+
+@pytest.fixture(autouse=True)
+def configdata_init():
+    """Initialize configdata if needed."""
+    if configdata.DATA is None:
+        configdata.init()
+
+
+def test_filter_languages():
+    filtered_langs = install_dict.filter_languages(LANGUAGE_LIST, ['af-ZA'])
+    assert filtered_langs == [AFRIKAANS]
+
+    filtered_langs = install_dict.filter_languages(
+        LANGUAGE_LIST, ['pl-PL', 'en-US'])
+    assert filtered_langs == [ENGLISH, POLISH]
+
+    with pytest.raises(install_dict.InvalidLanguageError):
+        install_dict.filter_languages(LANGUAGE_LIST, ['pl-PL', 'en-GB'])
+
+
+def test_install(tmpdir, monkeypatch):
+    monkeypatch.setattr(spell, 'dictionary_dir', lambda: str(tmpdir))
+    monkeypatch.setattr(
+        install_dict, 'download_dictionary',
+        lambda _url, dest: py.path.local(dest).ensure())  # pylint: disable=no-member
+    install_dict.install(LANGUAGE_LIST)
+    installed_files = [f.basename for f in tmpdir.listdir()]
+    expected_files = [lang.file_path for lang in LANGUAGE_LIST]
+    assert sorted(installed_files) == sorted(expected_files)