From f5d719dfd4410e996709a519f4ecb7ca976863a5 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Mon, 6 Nov 2017 00:51:41 -0600 Subject: [PATCH 1/5] importer: Chrome support This adds Chrome/Chromium support to the importer (which ought to be the last of these). Bookmarks are read from JSON, while keywords/search engines (the same thing here) are read from the Web Data sqlite3 database, and converted from OpenSearch format. importer: add tests for opensearch --- scripts/importer.py | 80 ++++++++++++++++++++++++++++- tests/unit/scripts/test_importer.py | 47 +++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/unit/scripts/test_importer.py diff --git a/scripts/importer.py b/scripts/importer.py index d664af7c1..f41a23b10 100755 --- a/scripts/importer.py +++ b/scripts/importer.py @@ -30,9 +30,13 @@ profiles is supported. import argparse import sqlite3 import os +import urllib.parse +import json +import string browser_default_input_format = { - 'chromium': 'netscape', + 'chromium': 'chrome', + 'chrome': 'chrome', 'ie': 'netscape', 'firefox': 'mozilla', 'seamonkey': 'mozilla', @@ -73,7 +77,8 @@ def main(): import_function = { 'netscape': import_netscape_bookmarks, - 'mozilla': import_moz_places + 'mozilla': import_moz_places, + 'chrome': import_chrome } import_function[input_format](args.bookmarks, bookmark_types, output_format) @@ -154,6 +159,33 @@ def search_escape(url): return url.replace('{', '{{').replace('}', '}}') +def opensearch_convert(url): + """Convert a basic OpenSearch URL into something Qutebrowser can use. + + Exceptions: + KeyError: + An unknown and required parameter is present in the URL. This + usually means there's browser/addon specific functionality needed + to build the URL (I'm looking at you and your browser, Google) that + obviously won't be present here. + """ + subst = { + 'searchTerms': '%s', # for proper escaping later + 'language': '*', + 'inputEncoding': 'UTF-8', + 'outputEncoding': 'UTF-8' + } + + # remove optional parameters (even those we don't support) + for param in string.Formatter().parse(url): + if param[1]: + if param[1].endswith('?'): + url = url.replace('{' + param[1] + '}', '') + elif param[2] and param[2].endswith('?'): + url = url.replace('{' + param[1] + ':' + param[2] + '}', '') + return search_escape(url.format(**subst)).replace('%s', '{}') + + def import_netscape_bookmarks(bookmarks_file, bookmark_types, output_format): """Import bookmarks from a NETSCAPE-Bookmark-file v1. @@ -268,5 +300,49 @@ def import_moz_places(profile, bookmark_types, output_format): print(out_template[output_format][typ].format(**row)) +def import_chrome(profile, bookmark_types, output_format): + """Import bookmarks and search keywords from Chrome-type profiles. + + On Chrome, keywords and search engines are the same thing and handled in + their own database table; bookmarks cannot have associated keywords. This + is why the dictionary lookups here are much simpler. + """ + out_template = { + 'bookmark': '{url} {name}', + 'quickmark': '{name} {url}', + 'search': "c.url.searchengines['{keyword}'] = '{url}'", + 'oldsearch': '{keyword} {url}' + } + + if 'search' in bookmark_types: + webdata = sqlite3.connect(os.path.join(profile, 'Web Data')) + c = webdata.cursor() + c.execute('SELECT keyword,url FROM keywords;') + for keyword, url in c: + try: + url = opensearch_convert(url) + print(out_template[output_format].format( + keyword=keyword, url=url)) + except KeyError: + print('# Unsupported parameter in url for {}; skipping....'. + format(keyword)) + + else: + with open(os.path.join(profile, 'Bookmarks'), encoding='utf-8') as f: + bookmarks = json.load(f) + + def bm_tree_walk(bm, template): + assert 'type' in bm + if bm['type'] == 'url': + if urllib.parse.urlparse(bm['url']).scheme != 'chrome': + print(template.format(**bm)) + elif bm['type'] == 'folder': + for child in bm['children']: + bm_tree_walk(child, template) + + for root in bookmarks['roots'].values(): + bm_tree_walk(root, out_template[output_format]) + + if __name__ == '__main__': main() diff --git a/tests/unit/scripts/test_importer.py b/tests/unit/scripts/test_importer.py new file mode 100644 index 000000000..992922361 --- /dev/null +++ b/tests/unit/scripts/test_importer.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +import pytest +from scripts import importer + + +def test_opensearch_convert(): + urls = [ + # simple search query + ('http://foo.bar/s?q={searchTerms}', 'http://foo.bar/s?q={}'), + # simple search query with supported additional parameter + ('http://foo.bar/s?q={searchTerms}&enc={inputEncoding}', + 'http://foo.bar/s?q={}&enc=UTF-8'), + # same as above but with supported optional parameter + ('http://foo.bar/s?q={searchTerms}&enc={inputEncoding?}', + 'http://foo.bar/s?q={}&enc='), + # unsupported-but-optional parameter + ('http://foo.bar/s?q={searchTerms}&opt={unsupported?}', + 'http://foo.bar/s?q={}&opt='), + # unsupported-but-optional subset parameter + ('http://foo.bar/s?q={searchTerms}&opt={unsupported:unsupported?}', + 'http://foo.bar/s?q={}&opt=') + ] + for os_url, qb_url in urls: + assert importer.opensearch_convert(os_url) == qb_url + # pass a required unsupported parameter + with pytest.raises(KeyError): + os_url = 'http://foo.bar/s?q={searchTerms}&req={unsupported}' + importer.opensearch_convert(os_url) From 8a695648d3f05e03c251955337d87c2a04d90a0e Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Wed, 8 Nov 2017 15:08:20 -0600 Subject: [PATCH 2/5] :%s/Qutebrowser/qutebrowser/g --- scripts/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/importer.py b/scripts/importer.py index f41a23b10..ab6ae1add 100755 --- a/scripts/importer.py +++ b/scripts/importer.py @@ -154,13 +154,13 @@ def get_args(): def search_escape(url): """Escape URLs such that preexisting { and } are handled properly. - Will obviously trash a properly-formatted Qutebrowser URL. + Will obviously trash a properly-formatted qutebrowser URL. """ return url.replace('{', '{{').replace('}', '}}') def opensearch_convert(url): - """Convert a basic OpenSearch URL into something Qutebrowser can use. + """Convert a basic OpenSearch URL into something qutebrowser can use. Exceptions: KeyError: From 2b7210f6d19801a58ba67e2264a5f9075a34d198 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Wed, 8 Nov 2017 15:11:07 -0600 Subject: [PATCH 3/5] importer: trailing commas --- scripts/importer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/importer.py b/scripts/importer.py index ab6ae1add..5f22646d6 100755 --- a/scripts/importer.py +++ b/scripts/importer.py @@ -40,7 +40,7 @@ browser_default_input_format = { 'ie': 'netscape', 'firefox': 'mozilla', 'seamonkey': 'mozilla', - 'palemoon': 'mozilla' + 'palemoon': 'mozilla', } @@ -78,7 +78,7 @@ def main(): import_function = { 'netscape': import_netscape_bookmarks, 'mozilla': import_moz_places, - 'chrome': import_chrome + 'chrome': import_chrome, } import_function[input_format](args.bookmarks, bookmark_types, output_format) @@ -332,7 +332,7 @@ def import_chrome(profile, bookmark_types, output_format): bookmarks = json.load(f) def bm_tree_walk(bm, template): - assert 'type' in bm + assert 'type' in bm, bm if bm['type'] == 'url': if urllib.parse.urlparse(bm['url']).scheme != 'chrome': print(template.format(**bm)) From 5688fc9910a99aadbe2ca8b1de012276960ef776 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Wed, 8 Nov 2017 15:13:16 -0600 Subject: [PATCH 4/5] importer: test unsupported opensearch separate --- tests/unit/scripts/test_importer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/scripts/test_importer.py b/tests/unit/scripts/test_importer.py index 992922361..c799af239 100644 --- a/tests/unit/scripts/test_importer.py +++ b/tests/unit/scripts/test_importer.py @@ -41,7 +41,10 @@ def test_opensearch_convert(): ] for os_url, qb_url in urls: assert importer.opensearch_convert(os_url) == qb_url - # pass a required unsupported parameter + + +def test_opensearch_convert_unsupported(): + """pass an unsupported, required parameter""" with pytest.raises(KeyError): os_url = 'http://foo.bar/s?q={searchTerms}&req={unsupported}' importer.opensearch_convert(os_url) From b5bf114ad4c3bb53a96b813a0cdd2837519b088c Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 9 Nov 2017 02:39:43 -0600 Subject: [PATCH 5/5] importer: add chrome profile tests --- tests/unit/scripts/chrome-profile/Bookmarks | 42 ++++++++++++++++++++ tests/unit/scripts/chrome-profile/Web Data | Bin 0 -> 63488 bytes tests/unit/scripts/test_importer.py | 25 +++++++++++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/unit/scripts/chrome-profile/Bookmarks create mode 100644 tests/unit/scripts/chrome-profile/Web Data diff --git a/tests/unit/scripts/chrome-profile/Bookmarks b/tests/unit/scripts/chrome-profile/Bookmarks new file mode 100644 index 000000000..69f6489f6 --- /dev/null +++ b/tests/unit/scripts/chrome-profile/Bookmarks @@ -0,0 +1,42 @@ +{ + "checksum": "8cfaaff489c8d353ed5fde89dbe373f2", + "roots": { + "bookmark_bar": { + "children": [ { + "date_added": "13154663015324557", + "id": "6", + "name": "Foo", + "type": "url", + "url": "http://foo.com/" + }, { + "date_added": "13154663025077469", + "id": "7", + "name": "Bar", + "type": "url", + "url": "http://bar.com/" + } ], + "date_added": "13154662986915782", + "date_modified": "13154663025077469", + "id": "1", + "name": "Bookmarks bar", + "type": "folder" + }, + "other": { + "children": [ ], + "date_added": "13154662986915792", + "date_modified": "0", + "id": "2", + "name": "Other bookmarks", + "type": "folder" + }, + "synced": { + "children": [ ], + "date_added": "13154662986915795", + "date_modified": "0", + "id": "3", + "name": "Mobile bookmarks", + "type": "folder" + } + }, + "version": 1 +} diff --git a/tests/unit/scripts/chrome-profile/Web Data b/tests/unit/scripts/chrome-profile/Web Data new file mode 100644 index 0000000000000000000000000000000000000000..d58b6dd9d665a8951d1050ae3bb533c72a202e5f GIT binary patch literal 63488 zcmeI5-*4N-9l%A&Hti&C>!z-nre4_^bz(q@EX%g627>3LP8!6q6aVOz1_eP;Cy6yh zGCV4dmwAJnb}z$ThW!b9?VqrRb$i*<_OiXM!?1_G4A_7T1M;wa?-nYQq8oRiX#;gqX|Jsd{>eZi9`~|ns2_OL^ zfCNr2f$yYn{{QrP7cD~qNB{{S0VL3m0M7sSBZG1zfCP{L5$QGb@CyIekObliTF9K1e`H3y)q@VFK9-U zJXTtcS<`e~Hk2mBk5u#)k#%#Q*j~b!>sxE9J8ROdjrZ2RFL~0Q?9%3jpD0~sg=M5b zX|o%nBk4<1LcO zri7)ZIPv;@KS~Na~YPJlg=g@pcy1RAj&g#~E>GsCe)Ninra$y%hr1n zV}(?y5gC>s;qJ1Dr7J4&%+8p$28{*cni}txa%5ttW);e% zo}L-ns@DmGQVpn?u|Ol!YU#|K0Zv^{c6=;X4rI&JHTA&LJ>YH4fQYL`xM1%4CTLL8 zV*O7K3{|eTpiZydDY`>!U_qFI0j-H*s|^{_npE!aYZ9mpeB>XiH5rhgVbnbgM4&eq zZ7H_W43yes5t|+`DL&n+FgDd7`!Y1qkU|nC@<^j_5_cP<4|?`6C8jT3651u+wjFbi z7&0X`G&d63849)XP?B`HbAZPsbT0Xg8(M~aV1X7gKU%uBzIko7G&GXFbE#if6RJke zhjxc+f|#D16xunid6PKMFFHyxq$iVHW*%GL+_>5CxPQ7yyBoLe?XCs2^?IQ|_Ss9x zk@VHcek!sWlDPT}S{4AOR$R1o{)e`TzcGFa-%90VIF~kicmr zFh2A{mrRD*d^#^Is`40dJN=4+ z+4ZUhi<>ES6-o*2orBP&fBc4>6J{TD0!Mpi#PmDw2%lcy{Xx*-!R@o5ULcrqYzM%4 zBN-k}vF7BRI2nmNwlo_aF+oDpG@ORl@IxsF@VKO#zzUGvfq*?na-SCP=LDT6d;jkl zd}60+MykMbUDo?_Iym+etMhze7U%y5{d_?EkN^@u0!RP}oIC*>|4*JNmWTw901`j~ zNMO(j;P^l29zy+)01`j~NB{|(JOLd4Po623hy;)T5Kmter37k9u9RE+ADVB%?kN^@u z0!U!c35+NIk{A{2#K^x`#Gllkl3xn{P5uG|_<2qQ+Gj36-2D}yJspmB1Mwn=E;|Pn zk$E&3KZHs&VJZ~)qHngoLEYVO{BB6=_RDD=I;g!8R)&WjXMv(242s0KHjw^lO`&>png|_Ro#g z_3gFUqdSvgI-3Qa4^mCN_XlpZG~0+`JcUB;!fudnAP( z$$~Fo!RZw}93?3>pi*o+r9P36Y@%wG#$r2#161*FVV%G(o(8k+tyjeK;-YYLGsu&N zfQhu6kSra?HIyUB)_3MfwwYr>zqI$yi|JxfcskCFY8g!h)`_UHB`f@ZOR>UNLuSQh z3EGv350&GsX&{~88RF{lm9eXiMqd`wg@W+3$SsetJ4Q4avN|S1$Y$1sx4NOkV#5v@ z#Mw9PJ+GBzc(}Bh+43XHWgsq5I3SUKp3m{LU`V1sl2lQQy-2%e1B(~EsGAkz|2(Hr z9*ac+NB{{S0VFWU1aSU;kiCPtApsMh{h#N<6-z||NB{{S0VFWU1n~ZU zkiCPtApsN!_@7MtIgvEr&*}R-J(rqFOipdL&yJ0cj;7mLVRU?ScVfIf zJw85OzBhUemPM+-_X@n%)NmY&zMISK@9$snWI38sa%@Mou4t+mp9R*1QX4A|%TFlZ z+97t69?smoUfwO_mu7BkZrv$ge}8N9&c^P}jqGx487BRhn#L7{T8~#AwupTYDa?C` zsrn&R^KdGB$T_^AILUy;-4`ZuP8@Aab zIfLxOyEZ?4F)1Z3ymjxBI(NsXxv|ls?C9=OadH>@@if)GKYEiF>J}aKN>`ezDUZM( z26Kw1M@6B8snIbb>JKm5*7}Es-nP4do}%Al$ZweTcB>9N*g9Z-7KKuc!uxhS%Qn2# zL;WPTq8l2~tB$QH`d!6_)x`jmpF`W+-sWv$y|WyjpW+^SU1x8gb%pTmk4kP;eSv1c zdqjq__Sj+T5LVX=rJ|E+Z#_W5A&;GwqPx|7Ky1p4&`9}-W>_s}&485$p&1|YCg_>Z zh;>X3fb&xOo2PQM%YH_eTF01~I+blLnz*G^4`&_&%d#`m&of-p9e7X4w_k19@S@*c zlWHNKfY=6ZccJm_+Pa_Hg8l385*5{In#xym^OIa#JS@KA@G2x=+A;S?*Ws2;YQ#|+ z+~J3`Rfktsqt>k6y>*)$^!mYTg8QV>uvRE{r&cYy=RmgO%^9!yU`wo-%=50hw+L*U1y@Ti^bOQ0q}teOI^20q>7;ElW3{=VR?2-f84A--~>2rvaw6(fsDEjwUQXp%Cu%0wN@B2LM zFlwHh3w5O{UV5+@Y}1hc^EfffY>s zg84l`H1CYAm20&jwNlA1)+)tpZGKtFE|3ab|JUYW*k4*!m(=CjT)qU?|Ly(Zi(u0| z)4dTYM2y;8S4`bAiroddqv@57G-^mp4#?@{IRSSXQFwpOgoWfvBfm$Id~#bP#37Uqg-sYvDuVENJO zLvJK5yt&(+c1>Rln|@zunC4pt>~DYAC=afzJHx&QqxP8NP%MjGSn?u)XHzk%;ewNkOUv`6!rXk8