Merge pull request #3257 from ryan-farley/import-chrome

importer: Chrome support

scripts/importer.py

@@ -30,13 +30,17 @@ profiles is supported.
 import argparse
 import sqlite3
 import os
+import urllib.parse
+import json
+import string
 
 browser_default_input_format = {
-    'chromium': 'netscape',
+    'chromium': 'chrome',
+    'chrome': 'chrome',
     'ie': 'netscape',
     'firefox': 'mozilla',
     'seamonkey': 'mozilla',
-    'palemoon': 'mozilla'
+    'palemoon': 'mozilla',
 }
 
 
@@ -73,7 +77,8 @@ def main():
 
     import_function = {
         'netscape': import_netscape_bookmarks,
-        'mozilla': import_moz_places
+        'mozilla': import_moz_places,
+        'chrome': import_chrome,
     }
     import_function[input_format](args.bookmarks, bookmark_types,
                                   output_format)
@@ -149,11 +154,38 @@ def get_args():
 def search_escape(url):
     """Escape URLs such that preexisting { and } are handled properly.
 
-    Will obviously trash a properly-formatted Qutebrowser URL.
+    Will obviously trash a properly-formatted qutebrowser URL.
     """
     return url.replace('{', '{{').replace('}', '}}')
 
 
+def opensearch_convert(url):
+    """Convert a basic OpenSearch URL into something qutebrowser can use.
+
+    Exceptions:
+        KeyError:
+            An unknown and required parameter is present in the URL. This
+            usually means there's browser/addon specific functionality needed
+            to build the URL (I'm looking at you and your browser, Google) that
+            obviously won't be present here.
+    """
+    subst = {
+        'searchTerms': '%s',  # for proper escaping later
+        'language': '*',
+        'inputEncoding': 'UTF-8',
+        'outputEncoding': 'UTF-8'
+    }
+
+    # remove optional parameters (even those we don't support)
+    for param in string.Formatter().parse(url):
+        if param[1]:
+            if param[1].endswith('?'):
+                url = url.replace('{' + param[1] + '}', '')
+            elif param[2] and param[2].endswith('?'):
+                url = url.replace('{' + param[1] + ':' + param[2] + '}', '')
+    return search_escape(url.format(**subst)).replace('%s', '{}')
+
+
 def import_netscape_bookmarks(bookmarks_file, bookmark_types, output_format):
     """Import bookmarks from a NETSCAPE-Bookmark-file v1.
 
@@ -268,5 +300,49 @@ def import_moz_places(profile, bookmark_types, output_format):
             print(out_template[output_format][typ].format(**row))
 
 
+def import_chrome(profile, bookmark_types, output_format):
+    """Import bookmarks and search keywords from Chrome-type profiles.
+
+    On Chrome, keywords and search engines are the same thing and handled in
+    their own database table; bookmarks cannot have associated keywords. This
+    is why the dictionary lookups here are much simpler.
+    """
+    out_template = {
+        'bookmark': '{url} {name}',
+        'quickmark': '{name} {url}',
+        'search': "c.url.searchengines['{keyword}'] = '{url}'",
+        'oldsearch': '{keyword} {url}'
+    }
+
+    if 'search' in bookmark_types:
+        webdata = sqlite3.connect(os.path.join(profile, 'Web Data'))
+        c = webdata.cursor()
+        c.execute('SELECT keyword,url FROM keywords;')
+        for keyword, url in c:
+            try:
+                url = opensearch_convert(url)
+                print(out_template[output_format].format(
+                    keyword=keyword, url=url))
+            except KeyError:
+                print('# Unsupported parameter in url for {}; skipping....'.
+                      format(keyword))
+
+    else:
+        with open(os.path.join(profile, 'Bookmarks'), encoding='utf-8') as f:
+            bookmarks = json.load(f)
+
+        def bm_tree_walk(bm, template):
+            assert 'type' in bm, bm
+            if bm['type'] == 'url':
+                if urllib.parse.urlparse(bm['url']).scheme != 'chrome':
+                    print(template.format(**bm))
+            elif bm['type'] == 'folder':
+                for child in bm['children']:
+                    bm_tree_walk(child, template)
+
+        for root in bookmarks['roots'].values():
+            bm_tree_walk(root, out_template[output_format])
+
+
 if __name__ == '__main__':
     main()

tests/unit/scripts/chrome-profile/Bookmarks

@@ -0,0 +1,42 @@
+{
+   "checksum": "8cfaaff489c8d353ed5fde89dbe373f2",
+   "roots": {
+      "bookmark_bar": {
+         "children": [ {
+            "date_added": "13154663015324557",
+            "id": "6",
+            "name": "Foo",
+            "type": "url",
+            "url": "http://foo.com/"
+         }, {
+            "date_added": "13154663025077469",
+            "id": "7",
+            "name": "Bar",
+            "type": "url",
+            "url": "http://bar.com/"
+         } ],
+         "date_added": "13154662986915782",
+         "date_modified": "13154663025077469",
+         "id": "1",
+         "name": "Bookmarks bar",
+         "type": "folder"
+      },
+      "other": {
+         "children": [  ],
+         "date_added": "13154662986915792",
+         "date_modified": "0",
+         "id": "2",
+         "name": "Other bookmarks",
+         "type": "folder"
+      },
+      "synced": {
+         "children": [  ],
+         "date_added": "13154662986915795",
+         "date_modified": "0",
+         "id": "3",
+         "name": "Mobile bookmarks",
+         "type": "folder"
+      }
+   },
+   "version": 1
+}

tests/unit/scripts/test_importer.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+from scripts import importer
+
+_chrome_profile = 'tests/unit/scripts/chrome-profile'
+
+
+def test_opensearch_convert():
+    urls = [
+        # simple search query
+        ('http://foo.bar/s?q={searchTerms}', 'http://foo.bar/s?q={}'),
+        # simple search query with supported additional parameter
+        ('http://foo.bar/s?q={searchTerms}&enc={inputEncoding}',
+         'http://foo.bar/s?q={}&enc=UTF-8'),
+        # same as above but with supported optional parameter
+        ('http://foo.bar/s?q={searchTerms}&enc={inputEncoding?}',
+         'http://foo.bar/s?q={}&enc='),
+        # unsupported-but-optional parameter
+        ('http://foo.bar/s?q={searchTerms}&opt={unsupported?}',
+         'http://foo.bar/s?q={}&opt='),
+        # unsupported-but-optional subset parameter
+        ('http://foo.bar/s?q={searchTerms}&opt={unsupported:unsupported?}',
+         'http://foo.bar/s?q={}&opt=')
+    ]
+    for os_url, qb_url in urls:
+        assert importer.opensearch_convert(os_url) == qb_url
+
+
+def test_opensearch_convert_unsupported():
+    """pass an unsupported, required parameter."""
+    with pytest.raises(KeyError):
+        os_url = 'http://foo.bar/s?q={searchTerms}&req={unsupported}'
+        importer.opensearch_convert(os_url)
+
+
+def test_chrome_bookmarks(capsys):
+    """Read sample bookmarks from chrome profile."""
+    expected = ('Foo http://foo.com/\n' 'Bar http://bar.com/\n')
+    importer.import_chrome(_chrome_profile, ['bookmark'], 'quickmark')
+    imported = capsys.readouterr()[0]
+    assert imported == expected
+
+
+def test_chrome_searches(capsys):
+    """Read sample searches from chrome profile."""
+    expected = (
+        "# Unsupported parameter in url for google.com; skipping....\n"
+        "c.url.searchengines['bing.com'] = 'https://www.bing.com/search?q={}&PC=U316&FORM=CHROMN'\n"
+        "c.url.searchengines['yahoo.com'] = 'https://search.yahoo.com/search?ei=UTF-8&fr=crmas&p={}'\n"
+        "c.url.searchengines['aol.com'] = 'https://search.aol.com/aol/search?q={}'\n"
+        "c.url.searchengines['ask.com'] = 'http://www.ask.com/web?q={}'\n")
+    importer.import_chrome(_chrome_profile, ['search'], 'search')
+    imported = capsys.readouterr()[0]
+    assert imported == expected