From 8cb6b832d1e1311ee4404a44ad2f471eaa00c884 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 00:24:59 -0400 Subject: [PATCH 1/8] script to import history data from other browsers --- scripts/hist_importer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/hist_importer.py diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py new file mode 100644 index 000000000..e69de29bb From c6d140a40a77345c484bc8557f84f97d33539bfe Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 00:26:47 -0400 Subject: [PATCH 2/8] adding script to import history data from other browsers --- scripts/hist_importer.py | 135 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index e69de29bb..5f4ead361 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# This file is part of qutebrowser. + +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + + +''' +Tool to import browser history data from other browsers. Although, safari +support is still on the way. +''' + + +import argparse +import sqlite3 +import sys + + +def parser(): + """Parse command line arguments.""" + description = 'This program is meant to extract browser history from your'\ + 'previous browser and import them into qutebrowser.' + epilog = 'Databases:\n\tQute: Is named "history.sqlite" and can be found '\ + 'at your --basedir. In order to find where your basedir is you '\ + 'can run ":open qute:version" inside qutebrowser.'\ + '\n\tFirerox: Is named "places.sqlite", and can be found at your'\ + 'system\'s profile folder. Check this link for where it is locat'\ + 'ed: http://kb.mozillazine.org/Profile_folder'\ + '\n\tChrome: Is named "History", and can be found at the respec'\ + 'tive User Data Directory. Check this link for where it is locat'\ + 'ed: https://chromium.googlesource.com/chromium/src/+/master/'\ + 'docs/user_data_dir.md\n\n'\ + 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ + 'sqlite -d /qutebrowser/data/history.sqlite' + parser = argparse.ArgumentParser( + description=description, epilog=epilog, + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome, safari}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: fullpath to the sqlite data' + 'base file from the source browser.') + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='Destination: The fullpath to the qutebrowser ' + 'sqlite database') + return parser.parse_args() + + +def open_db(db): + """Open connection with database.""" + try: + conn = sqlite3.connect(db) + return conn + except Exception as e: + print('Error: {}'.format(e)) + raise('Error: There was some error trying to to connect with the [{}]' + 'database. Verify if the filepath is correct or is being used.'. + format(db)) + + +def extract(source, query): + """Performs extraction of (datetime,url,title) from source.""" + try: + conn = open_db(source) + cursor = conn.cursor() + cursor.execute(query) + history = cursor.fetchall() + conn.close() + return history + except Exception as e: + # print('Error: {}'.format(e)) + print(type(source)) + raise('Error: There was some error trying to to connect with the [{}]' + 'database. Verify if the filepath is correct or is being used.'. + format(str(source))) + + +def clean(history): + """Receives a list of records:(datetime,url,title). And clean all records + in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser + will throw errors.""" + nulls = [record for record in history if record[0] is None] + for null_datetime in nulls: + history.remove(null_datetime) + return history + + +def insert_qb(history, dest): + conn = open_db(dest) + cursor = conn.cursor() + cursor.executemany( + 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history + ) + conn.commit() + conn.close() + + +def main(): + args = parser() + browser = args.browser.lower() + source, dest = args.source, args.dest + query = { + 'firefox': 'select url,title,last_visit_date/1000000 as date ' + 'from moz_places', + 'chrome': 'select url,title,last_visit_time/10000000 as date ' + 'from urls', + 'safari': None + } + if browser not in query: + sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( + browser)) + else: + if browser == 'safari': + print('Sorry, currently we do not support this browser.') + sys.exit(1) + history = extract(source, query[browser]) + history = clean(history) + insert_qb(history, dest) + + +if __name__ == "__main__": + main() From 4dc232f259d4a24da075e584998998873e58c221 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 13:54:24 -0400 Subject: [PATCH 3/8] pylint fixes --- scripts/hist_importer.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 5f4ead361..44a76bd8a 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -17,10 +17,8 @@ # along with qutebrowser. If not, see . -''' -Tool to import browser history data from other browsers. Although, safari -support is still on the way. -''' +'''Tool to import browser history data from other browsers. Although, safari +support is still on the way.''' import argparse @@ -44,31 +42,31 @@ def parser(): 'docs/user_data_dir.md\n\n'\ 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ 'sqlite -d /qutebrowser/data/history.sqlite' - parser = argparse.ArgumentParser( + parsed = argparse.ArgumentParser( description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter ) - parser.add_argument('-b', '--browser', dest='browser', required=True, + parsed.add_argument('-b', '--browser', dest='browser', required=True, type=str, help='Browsers: {firefox, chrome, safari}') - parser.add_argument('-s', '--source', dest='source', required=True, + parsed.add_argument('-s', '--source', dest='source', required=True, type=str, help='Source: fullpath to the sqlite data' 'base file from the source browser.') - parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + parsed.add_argument('-d', '--dest', dest='dest', required=True, type=str, help='Destination: The fullpath to the qutebrowser ' 'sqlite database') - return parser.parse_args() + return parsed.parse_args() -def open_db(db): +def open_db(data_base): """Open connection with database.""" try: - conn = sqlite3.connect(db) + conn = sqlite3.connect(data_base) return conn - except Exception as e: - print('Error: {}'.format(e)) + except Exception as any_e: + print('Error: {}'.format(any_e)) raise('Error: There was some error trying to to connect with the [{}]' 'database. Verify if the filepath is correct or is being used.'. - format(db)) + format(data_base)) def extract(source, query): @@ -80,8 +78,8 @@ def extract(source, query): history = cursor.fetchall() conn.close() return history - except Exception as e: - # print('Error: {}'.format(e)) + except Exception as any_e: + print('Error: {}'.format(any_e)) print(type(source)) raise('Error: There was some error trying to to connect with the [{}]' 'database. Verify if the filepath is correct or is being used.'. @@ -99,6 +97,8 @@ def clean(history): def insert_qb(history, dest): + """Given a list of records in history and a dest db, insert all records in + the dest db.""" conn = open_db(dest) cursor = conn.cursor() cursor.executemany( @@ -109,19 +109,20 @@ def insert_qb(history, dest): def main(): + """Main control flux of the script.""" args = parser() browser = args.browser.lower() source, dest = args.source, args.dest query = { 'firefox': 'select url,title,last_visit_date/1000000 as date ' - 'from moz_places', + 'from moz_places', 'chrome': 'select url,title,last_visit_time/10000000 as date ' - 'from urls', + 'from urls', 'safari': None } if browser not in query: sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( - browser)) + browser)) else: if browser == 'safari': print('Sorry, currently we do not support this browser.') From 665a76561ecf405783f13beca694085a0803e479 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 22:50:52 -0400 Subject: [PATCH 4/8] add insertions to ComandHistory table as well --- scripts/hist_importer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 44a76bd8a..8df16384a 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -104,6 +104,10 @@ def insert_qb(history, dest): cursor.executemany( 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history ) + cursor.executemany( + 'INSERT INTO CompletionHistory (url,title,last_atime) VALUES (?,?,?)', + history + ) conn.commit() conn.close() From 92f9a8503ee90cba87bc11b9aa983e8c8069c497 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Tue, 3 Oct 2017 01:55:31 -0400 Subject: [PATCH 5/8] add required redirect (url,title,atime,redirect) --- scripts/hist_importer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 8df16384a..d96284879 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -89,10 +89,14 @@ def extract(source, query): def clean(history): """Receives a list of records:(datetime,url,title). And clean all records in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser - will throw errors.""" + will throw errors. Also, will add a 4rth attribute of '0' for the redirect + field in history.sqlite in qutebrowser.""" nulls = [record for record in history if record[0] is None] for null_datetime in nulls: history.remove(null_datetime) + history = [list(record) for record in history] + for record in history: + record.append('0') return history @@ -102,12 +106,10 @@ def insert_qb(history, dest): conn = open_db(dest) cursor = conn.cursor() cursor.executemany( - 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history - ) - cursor.executemany( - 'INSERT INTO CompletionHistory (url,title,last_atime) VALUES (?,?,?)', + 'INSERT INTO History (url,title,atime,redirect) VALUES (?,?,?,?)', history ) + cursor.execute('DROP TABLE CompletionHistory') conn.commit() conn.close() From 96599b96846aa108698946eb7d49ad7dc33581f0 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Fri, 17 Nov 2017 02:38:56 -0300 Subject: [PATCH 6/8] revisions set by The Compiler --- scripts/hist_importer.py | 95 ++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 48 deletions(-) mode change 100644 => 100755 scripts/hist_importer.py diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py old mode 100644 new mode 100755 index d96284879..f27b4e267 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# This file is part of qutebrowser. +# Copyright 2017 Florian Bruhin (The Compiler) +# Copyright 2014-2017 Josefson Souza +# This file is part of qutebrowser. +# # qutebrowser is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -17,60 +20,60 @@ # along with qutebrowser. If not, see . -'''Tool to import browser history data from other browsers. Although, safari -support is still on the way.''' +"""Tool to import browser history from other browsers.""" import argparse import sqlite3 import sys +import os -def parser(): +def parse(): """Parse command line arguments.""" - description = 'This program is meant to extract browser history from your'\ - 'previous browser and import them into qutebrowser.' - epilog = 'Databases:\n\tQute: Is named "history.sqlite" and can be found '\ - 'at your --basedir. In order to find where your basedir is you '\ - 'can run ":open qute:version" inside qutebrowser.'\ - '\n\tFirerox: Is named "places.sqlite", and can be found at your'\ - 'system\'s profile folder. Check this link for where it is locat'\ - 'ed: http://kb.mozillazine.org/Profile_folder'\ - '\n\tChrome: Is named "History", and can be found at the respec'\ - 'tive User Data Directory. Check this link for where it is locat'\ - 'ed: https://chromium.googlesource.com/chromium/src/+/master/'\ - 'docs/user_data_dir.md\n\n'\ - 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ - 'sqlite -d /qutebrowser/data/history.sqlite' - parsed = argparse.ArgumentParser( + description = ("This program is meant to extract browser history from your" + "previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\tQutebrowser: Is named 'history.sqlite' and can be" + " found at your --basedir. In order to find where your basedir" + " is you can run ':open qute:version' inside qutebrowser." + "\n\tFirerox: Is named 'places.sqlite', and can be found at your" + "system\"s profile folder. Check this link for where it is locat" + "ed: http://kb.mozillazine.org/Profile_folder" + "\n\tChrome: Is named 'History', and can be found at the respec" + "tive User Data Directory. Check this link for where it is locat" + "ed: https://chromium.googlesource.com/chromium/src/+/master/" + "docs/user_data_dir.md\n\n" + "Example: hist_importer.py -b firefox -s /Firefox/Profile/" + "places.sqlite -d /qutebrowser/data/history.sqlite") + parser = argparse.ArgumentParser( description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter ) - parsed.add_argument('-b', '--browser', dest='browser', required=True, - type=str, help='Browsers: {firefox, chrome, safari}') - parsed.add_argument('-s', '--source', dest='source', required=True, - type=str, help='Source: fullpath to the sqlite data' + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: Full path to the sqlite data' 'base file from the source browser.') - parsed.add_argument('-d', '--dest', dest='dest', required=True, type=str, - help='Destination: The fullpath to the qutebrowser ' + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='Destination: The full path to the qutebrowser ' 'sqlite database') - return parsed.parse_args() + return parser.parse_args() def open_db(data_base): """Open connection with database.""" - try: + if os.path.isfile(data_base): conn = sqlite3.connect(data_base) return conn - except Exception as any_e: - print('Error: {}'.format(any_e)) - raise('Error: There was some error trying to to connect with the [{}]' - 'database. Verify if the filepath is correct or is being used.'. - format(data_base)) + else: + raise sys.exit('\nDataBaseNotFound: There was some error trying to to' + ' connect with the [{}] database. Verify if the' + ' filepath is correct or is being used.' + .format(data_base)) def extract(source, query): - """Performs extraction of (datetime,url,title) from source.""" + """Extracts (datetime,url,title) from source database.""" try: conn = open_db(source) cursor = conn.cursor() @@ -78,18 +81,17 @@ def extract(source, query): history = cursor.fetchall() conn.close() return history - except Exception as any_e: - print('Error: {}'.format(any_e)) - print(type(source)) - raise('Error: There was some error trying to to connect with the [{}]' - 'database. Verify if the filepath is correct or is being used.'. - format(str(source))) + except sqlite3.OperationalError as op_e: + print('\nCould not perform queries into the source database: {}' + '\nBrowser version is not supported as it have a different sql' + ' schema.'.format(op_e)) def clean(history): - """Receives a list of records:(datetime,url,title). And clean all records - in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser - will throw errors. Also, will add a 4rth attribute of '0' for the redirect + """Clean up records from source database. + Receives a list of records:(datetime,url,title). And clean all records + in place, that has a NULL/None datetime attribute. Otherwise qutebrowser + will throw errors. Also, will add a 4th attribute of '0' for the redirect field in history.sqlite in qutebrowser.""" nulls = [record for record in history if record[0] is None] for null_datetime in nulls: @@ -101,7 +103,8 @@ def clean(history): def insert_qb(history, dest): - """Given a list of records in history and a dest db, insert all records in + """Insert history into dest database + Given a list of records in history and a dest db, insert all records in the dest db.""" conn = open_db(dest) cursor = conn.cursor() @@ -116,7 +119,7 @@ def insert_qb(history, dest): def main(): """Main control flux of the script.""" - args = parser() + args = parse() browser = args.browser.lower() source, dest = args.source, args.dest query = { @@ -124,15 +127,11 @@ def main(): 'from moz_places', 'chrome': 'select url,title,last_visit_time/10000000 as date ' 'from urls', - 'safari': None } if browser not in query: sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( browser)) else: - if browser == 'safari': - print('Sorry, currently we do not support this browser.') - sys.exit(1) history = extract(source, query[browser]) history = clean(history) insert_qb(history, dest) From 3131d3d3bc8acf7ba40029ffcfe59f2454d0bfbc Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Fri, 17 Nov 2017 11:48:34 -0300 Subject: [PATCH 7/8] Flake8 warnings pointed by travis. --- scripts/hist_importer.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index f27b4e267..79c723c58 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -73,7 +73,15 @@ def open_db(data_base): def extract(source, query): - """Extracts (datetime,url,title) from source database.""" + """Get records from source database. + + Args: + source: File path to the source database where we want to extract the + data from. + query: The query string to be executed in order to retrieve relevant + attributes as (datetime, url, time) from the source database according + to the browser chosen. + """ try: conn = open_db(source) cursor = conn.cursor() @@ -89,10 +97,15 @@ def extract(source, query): def clean(history): """Clean up records from source database. - Receives a list of records:(datetime,url,title). And clean all records - in place, that has a NULL/None datetime attribute. Otherwise qutebrowser - will throw errors. Also, will add a 4th attribute of '0' for the redirect - field in history.sqlite in qutebrowser.""" + + Receives a list of record and sanityze them in order for them to be + properly imported to qutebrowser. Sanitation requires addiing a 4th + attribute 'redirect' which is filled with '0's, and also purging all + records that have a NULL/None datetime attribute. + + Args: + history: List of records (datetime, url, title) from source database. + """ nulls = [record for record in history if record[0] is None] for null_datetime in nulls: history.remove(null_datetime) @@ -103,9 +116,13 @@ def clean(history): def insert_qb(history, dest): - """Insert history into dest database - Given a list of records in history and a dest db, insert all records in - the dest db.""" + """Insert history into dest database. + + Args: + history: List of records. + dest: File path to the destination database, where history will be + inserted. + """ conn = open_db(dest) cursor = conn.cursor() cursor.executemany( From b6466b74108d5087f3b5659d75dd4ab0e0e1ac93 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 4 Dec 2017 13:08:56 -0300 Subject: [PATCH 8/8] revision 2 --- scripts/hist_importer.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 79c723c58..36f151e6f 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -2,7 +2,7 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: # Copyright 2017 Florian Bruhin (The Compiler) -# Copyright 2014-2017 Josefson Souza +# Copyright 2017 Josefson Souza # This file is part of qutebrowser. # @@ -32,17 +32,17 @@ import os def parse(): """Parse command line arguments.""" description = ("This program is meant to extract browser history from your" - "previous browser and import them into qutebrowser.") - epilog = ("Databases:\n\tQutebrowser: Is named 'history.sqlite' and can be" + " previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can be" " found at your --basedir. In order to find where your basedir" " is you can run ':open qute:version' inside qutebrowser." - "\n\tFirerox: Is named 'places.sqlite', and can be found at your" - "system\"s profile folder. Check this link for where it is locat" - "ed: http://kb.mozillazine.org/Profile_folder" - "\n\tChrome: Is named 'History', and can be found at the respec" - "tive User Data Directory. Check this link for where it is locat" - "ed: https://chromium.googlesource.com/chromium/src/+/master/" - "docs/user_data_dir.md\n\n" + "\n\n\tFirefox: Is named 'places.sqlite', and can be found at your" + " system's profile folder. Check this link for where it is " + "located: http://kb.mozillazine.org/Profile_folder" + "\n\n\tChrome: Is named 'History', and can be found at the " + "respective User Data Directory. Check this link for where it is" + "located: https://chromium.googlesource.com/chromium/src/+/" + "master/docs/user_data_dir.md\n\n" "Example: hist_importer.py -b firefox -s /Firefox/Profile/" "places.sqlite -d /qutebrowser/data/history.sqlite") parser = argparse.ArgumentParser( @@ -55,7 +55,7 @@ def parse(): type=str, help='Source: Full path to the sqlite data' 'base file from the source browser.') parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, - help='Destination: The full path to the qutebrowser ' + help='\nDestination: Full path to the qutebrowser ' 'sqlite database') return parser.parse_args() @@ -66,10 +66,9 @@ def open_db(data_base): conn = sqlite3.connect(data_base) return conn else: - raise sys.exit('\nDataBaseNotFound: There was some error trying to to' + raise sys.exit('DataBaseNotFound: There was some error trying to to' ' connect with the [{}] database. Verify if the' - ' filepath is correct or is being used.' - .format(data_base)) + ' given file path is correct.'.format(data_base)) def extract(source, query): @@ -91,8 +90,7 @@ def extract(source, query): return history except sqlite3.OperationalError as op_e: print('\nCould not perform queries into the source database: {}' - '\nBrowser version is not supported as it have a different sql' - ' schema.'.format(op_e)) + .format(op_e)) def clean(history):