From dfbcb7531368f6791ef83f5d96aa57308e9eab91 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 22 Jan 2017 21:17:50 +0100 Subject: [PATCH 1/6] First prototype of QtWebKit-NG history/session support --- qutebrowser/browser/webkit/tabhistory.py | 76 ++++++++++++++++++++---- scripts/testbrowser.py | 12 +++- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/qutebrowser/browser/webkit/tabhistory.py b/qutebrowser/browser/webkit/tabhistory.py index 63a973685..cf9cc9f10 100644 --- a/qutebrowser/browser/webkit/tabhistory.py +++ b/qutebrowser/browser/webkit/tabhistory.py @@ -21,21 +21,41 @@ from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl +from PyQt5.QtWebKit import qWebKitVersion # FIXME can we guarantee WebKit is available here? from qutebrowser.utils import qtutils -HISTORY_STREAM_VERSION = 2 -BACK_FORWARD_TREE_VERSION = 2 - - def _encode_url(url): """Encode a QUrl suitable to pass to QWebHistory.""" data = bytes(QUrl.toPercentEncoding(url.toString(), b':/#?&+=@%*')) return data.decode('ascii') -def _serialize_item(i, item, stream): +def _serialize_item_ng(i, item): + data = { + 'children': [], + 'documentSequenceNumber': i + 1, # FIXME what to pass here? + 'documentState': [], + 'formContentType': '', + 'itemSequenceNumber': i + 1, # FIXME what to pass here? + 'originalURLString': item.original_url.toString(), # FIXME encoding? + 'pageScaleFactor': 0.0, + 'referrer': '', + 'scrollPosition': {'x': 0, 'y': 0}, + 'target': '', + 'title': item.title, + 'urlString': item.url.toString(), # FIXME encoding? + } + try: + data['scrollPosition']['x'] = item.user_data['scroll-pos'].x() + data['scrollPosition']['y'] = item.user_data['scroll-pos'].y() + except (KeyError, TypeError): + pass + return data + + +def _serialize_item_old(i, item, stream): """Serialize a single WebHistoryItem into a QDataStream. Args: @@ -53,7 +73,7 @@ def _serialize_item(i, item, stream): ### Source/WebCore/history/HistoryItem.cpp decodeBackForwardTree ## backForwardTreeEncodingVersion - stream.writeUInt32(BACK_FORWARD_TREE_VERSION) + stream.writeUInt32(2) ## size (recursion stack) stream.writeUInt64(0) ## node->m_documentSequenceNumber @@ -101,6 +121,39 @@ def _serialize_item(i, item, stream): stream.writeBool(False) +def _serialize_old(items, current_idx, stream): + ### Source/WebKit/qt/Api/qwebhistory.cpp operator<< + stream.writeInt(2) # history stream version + stream.writeInt(len(items)) + stream.writeInt(current_idx) + + for i, item in enumerate(items): + _serialize_item_old(i, item, stream) + user_data.append(item.user_data) + + +def _serialize_ng(items, current_idx, stream): + # {'currentItemIndex': 0, + # 'history': [{'children': [], + # 'documentSequenceNumber': 1485030525573123, + # 'documentState': [], + # 'formContentType': '', + # 'itemSequenceNumber': 1485030525573122, + # 'originalURLString': 'about:blank', + # 'pageScaleFactor': 0.0, + # 'referrer': '', + # 'scrollPosition': {'x': 0, 'y': 0}, + # 'target': '', + # 'title': '', + # 'urlString': 'about:blank'}]} + data = {'currentItemIndex': current_idx, 'history': []} + for i, item in enumerate(items): + data['history'].append(_serialize_item_ng(i, item)) + + stream.writeInt(3) # history stream version + stream.writeQVariantMap(data) + + def serialize(items): """Serialize a list of QWebHistoryItems to a data stream. @@ -137,13 +190,12 @@ def serialize(items): else: current_idx = 0 - ### Source/WebKit/qt/Api/qwebhistory.cpp operator<< - stream.writeInt(HISTORY_STREAM_VERSION) - stream.writeInt(len(items)) - stream.writeInt(current_idx) + if qWebKitVersion() == '538.1': # FIXME better comparison + _serialize_old(items, current_idx, stream) + else: + _serialize_ng(items, current_idx, stream) - for i, item in enumerate(items): - _serialize_item(i, item, stream) + for i, item in enumerate(items): # FIXME easier way? user_data.append(item.user_data) stream.device().reset() diff --git a/scripts/testbrowser.py b/scripts/testbrowser.py index 24ce2a130..1abd92bfa 100755 --- a/scripts/testbrowser.py +++ b/scripts/testbrowser.py @@ -23,7 +23,7 @@ import sys import argparse -from PyQt5.QtCore import QUrl +from PyQt5.QtCore import QUrl, QDataStream, QIODevice, QByteArray from PyQt5.QtWidgets import QApplication try: @@ -49,6 +49,15 @@ def parse_args(): return parser.parse_args() +def dump_history(wv): + ba = QByteArray() + ds = QDataStream(ba, QIODevice.ReadWrite) + ds << wv.history() + ds2 = QDataStream(ba) + assert ds2.readInt() == 3 + print(ds2.readQVariantMap()) + + if __name__ == '__main__': args = parse_args() app = QApplication(sys.argv) @@ -73,6 +82,7 @@ if __name__ == '__main__': wv.loadStarted.connect(lambda: print("Loading started")) wv.loadProgress.connect(lambda p: print("Loading progress: {}%".format(p))) wv.loadFinished.connect(lambda: print("Loading finished")) + wv.loadFinished.connect(lambda: dump_history(wv)) if args.plugins and not using_webengine: from PyQt5.QtWebKit import QWebSettings From 99d163687874f366f9e9e1e43eef6b1b36279f5b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 5 Feb 2017 20:17:57 +0100 Subject: [PATCH 2/6] Properly implement session support for QtWebKit-NG See #1571 --- qutebrowser/browser/webkit/tabhistory.py | 90 +++++++++++------------- qutebrowser/utils/qtutils.py | 10 +++ scripts/testbrowser.py | 12 +--- tests/unit/utils/test_qtutils.py | 9 +++ 4 files changed, 60 insertions(+), 61 deletions(-) diff --git a/qutebrowser/browser/webkit/tabhistory.py b/qutebrowser/browser/webkit/tabhistory.py index cf9cc9f10..d2efca257 100644 --- a/qutebrowser/browser/webkit/tabhistory.py +++ b/qutebrowser/browser/webkit/tabhistory.py @@ -21,7 +21,7 @@ from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl -from PyQt5.QtWebKit import qWebKitVersion # FIXME can we guarantee WebKit is available here? +from PyQt5.QtWebKit import qWebKitVersion from qutebrowser.utils import qtutils @@ -32,20 +32,34 @@ def _encode_url(url): return data.decode('ascii') -def _serialize_item_ng(i, item): +def _serialize_ng(items, current_idx, stream): + # {'currentItemIndex': 0, + # 'history': [{'children': [], + # 'documentSequenceNumber': 1485030525573123, + # 'documentState': [], + # 'formContentType': '', + # 'itemSequenceNumber': 1485030525573122, + # 'originalURLString': 'about:blank', + # 'pageScaleFactor': 0.0, + # 'referrer': '', + # 'scrollPosition': {'x': 0, 'y': 0}, + # 'target': '', + # 'title': '', + # 'urlString': 'about:blank'}]} + data = {'currentItemIndex': current_idx, 'history': []} + for item in items: + data['history'].append(_serialize_item_ng(item)) + + stream.writeInt(3) # history stream version + stream.writeQVariantMap(data) + + +def _serialize_item_ng(item): data = { - 'children': [], - 'documentSequenceNumber': i + 1, # FIXME what to pass here? - 'documentState': [], - 'formContentType': '', - 'itemSequenceNumber': i + 1, # FIXME what to pass here? - 'originalURLString': item.original_url.toString(), # FIXME encoding? - 'pageScaleFactor': 0.0, - 'referrer': '', + 'originalURLString': item.original_url.toString(QUrl.FullyEncoded), 'scrollPosition': {'x': 0, 'y': 0}, - 'target': '', 'title': item.title, - 'urlString': item.url.toString(), # FIXME encoding? + 'urlString': item.url.toString(QUrl.FullyEncoded), } try: data['scrollPosition']['x'] = item.user_data['scroll-pos'].x() @@ -55,6 +69,16 @@ def _serialize_item_ng(i, item): return data +def _serialize_old(items, current_idx, stream): + ### Source/WebKit/qt/Api/qwebhistory.cpp operator<< + stream.writeInt(2) # history stream version + stream.writeInt(len(items)) + stream.writeInt(current_idx) + + for i, item in enumerate(items): + _serialize_item_old(i, item, stream) + + def _serialize_item_old(i, item, stream): """Serialize a single WebHistoryItem into a QDataStream. @@ -121,39 +145,6 @@ def _serialize_item_old(i, item, stream): stream.writeBool(False) -def _serialize_old(items, current_idx, stream): - ### Source/WebKit/qt/Api/qwebhistory.cpp operator<< - stream.writeInt(2) # history stream version - stream.writeInt(len(items)) - stream.writeInt(current_idx) - - for i, item in enumerate(items): - _serialize_item_old(i, item, stream) - user_data.append(item.user_data) - - -def _serialize_ng(items, current_idx, stream): - # {'currentItemIndex': 0, - # 'history': [{'children': [], - # 'documentSequenceNumber': 1485030525573123, - # 'documentState': [], - # 'formContentType': '', - # 'itemSequenceNumber': 1485030525573122, - # 'originalURLString': 'about:blank', - # 'pageScaleFactor': 0.0, - # 'referrer': '', - # 'scrollPosition': {'x': 0, 'y': 0}, - # 'target': '', - # 'title': '', - # 'urlString': 'about:blank'}]} - data = {'currentItemIndex': current_idx, 'history': []} - for i, item in enumerate(items): - data['history'].append(_serialize_item_ng(i, item)) - - stream.writeInt(3) # history stream version - stream.writeQVariantMap(data) - - def serialize(items): """Serialize a list of QWebHistoryItems to a data stream. @@ -190,13 +181,12 @@ def serialize(items): else: current_idx = 0 - if qWebKitVersion() == '538.1': # FIXME better comparison - _serialize_old(items, current_idx, stream) - else: + if qtutils.is_qtwebkit_ng(qWebKitVersion()): _serialize_ng(items, current_idx, stream) + else: + _serialize_old(items, current_idx, stream) - for i, item in enumerate(items): # FIXME easier way? - user_data.append(item.user_data) + user_data += [item.user_data for item in items] stream.device().reset() qtutils.check_qdatastream(stream) diff --git a/qutebrowser/utils/qtutils.py b/qutebrowser/utils/qtutils.py index c573628e1..4e4d7ce05 100644 --- a/qutebrowser/utils/qtutils.py +++ b/qutebrowser/utils/qtutils.py @@ -91,6 +91,16 @@ def version_check(version, op=operator.ge): pkg_resources.parse_version(version)) +def is_qtwebkit_ng(version): + """Check if the given version is QtWebKit-NG. + + This is typically used as is_webkit_ng(qWebKitVersion) but we don't want to + have QtWebKit imports in here. + """ + return (pkg_resources.parse_version(version) > + pkg_resources.parse_version('538.1')) + + def check_overflow(arg, ctype, fatal=True): """Check if the given argument is in bounds for the given type. diff --git a/scripts/testbrowser.py b/scripts/testbrowser.py index 1abd92bfa..24ce2a130 100755 --- a/scripts/testbrowser.py +++ b/scripts/testbrowser.py @@ -23,7 +23,7 @@ import sys import argparse -from PyQt5.QtCore import QUrl, QDataStream, QIODevice, QByteArray +from PyQt5.QtCore import QUrl from PyQt5.QtWidgets import QApplication try: @@ -49,15 +49,6 @@ def parse_args(): return parser.parse_args() -def dump_history(wv): - ba = QByteArray() - ds = QDataStream(ba, QIODevice.ReadWrite) - ds << wv.history() - ds2 = QDataStream(ba) - assert ds2.readInt() == 3 - print(ds2.readQVariantMap()) - - if __name__ == '__main__': args = parse_args() app = QApplication(sys.argv) @@ -82,7 +73,6 @@ if __name__ == '__main__': wv.loadStarted.connect(lambda: print("Loading started")) wv.loadProgress.connect(lambda p: print("Loading progress: {}%".format(p))) wv.loadFinished.connect(lambda: print("Loading finished")) - wv.loadFinished.connect(lambda: dump_history(wv)) if args.plugins and not using_webengine: from PyQt5.QtWebKit import QWebSettings diff --git a/tests/unit/utils/test_qtutils.py b/tests/unit/utils/test_qtutils.py index 094e9a1aa..db2767a82 100644 --- a/tests/unit/utils/test_qtutils.py +++ b/tests/unit/utils/test_qtutils.py @@ -64,6 +64,15 @@ def test_version_check(monkeypatch, qversion, version, op, expected): assert qtutils.version_check(version, op) == expected +@pytest.mark.parametrize('version, ng', [ + ('537.21', False), # QtWebKit 5.1 + ('538.1', False), # Qt 5.8 + ('602.1', True) # QtWebKit-NG TP5 +]) +def test_is_qtwebkit_ng(version, ng): + assert qtutils.is_qtwebkit_ng(version) == ng + + class TestCheckOverflow: """Test check_overflow.""" From f0f97a52134525f2842952ebd6ab7c3bba6ea76c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 5 Feb 2017 20:21:42 +0100 Subject: [PATCH 3/6] Add QtWebKit-NG test job to Travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 09fcacc33..5991fea42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,9 @@ matrix: - os: linux env: DOCKER=archlinux QUTE_BDD_WEBENGINE=true services: docker + - os: linux + env: DOCKER=archlinux-ng + services: docker - os: linux env: DOCKER=ubuntu-xenial services: docker From 91bdc00410ae91e0d7e5dc395dd3c72ed5bdb9e1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 5 Feb 2017 20:32:29 +0100 Subject: [PATCH 4/6] Make tests work with QtWebKit-NG Fixes #1571 --- pytest.ini | 2 ++ tests/end2end/conftest.py | 18 +++++++++++++++++- tests/end2end/features/downloads.feature | 2 +- tests/end2end/features/history.feature | 2 +- tests/end2end/features/javascript.feature | 1 + tests/end2end/features/misc.feature | 4 ++-- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index 2285053a9..8975d6b84 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,6 +17,8 @@ markers = qtwebengine_todo: Features still missing with QtWebEngine qtwebengine_skip: Tests not applicable with QtWebEngine qtwebkit_skip: Tests not applicable with QtWebKit + qtwebkit_ng_xfail: Tests failing with QtWebKit-NG + qtwebkit_ng_skip: Tests skipped with QtWebKit-NG qtwebengine_flaky: Tests which are flaky (and currently skipped) with QtWebEngine qtwebengine_osx_xfail: Tests which fail on OS X with QtWebEngine js_prompt: Tests needing to display a javascript prompt diff --git a/tests/end2end/conftest.py b/tests/end2end/conftest.py index d2512457e..2219f2720 100644 --- a/tests/end2end/conftest.py +++ b/tests/end2end/conftest.py @@ -106,7 +106,9 @@ def _get_backend_tag(tag): pytest_marks = { 'qtwebengine_todo': pytest.mark.qtwebengine_todo, 'qtwebengine_skip': pytest.mark.qtwebengine_skip, - 'qtwebkit_skip': pytest.mark.qtwebkit_skip + 'qtwebkit_skip': pytest.mark.qtwebkit_skip, + 'qtwebkit_ng_xfail': pytest.mark.qtwebkit_ng_xfail, + 'qtwebkit_ng_skip': pytest.mark.qtwebkit_ng_skip, } if not any(tag.startswith(t + ':') for t in pytest_marks): return None @@ -132,6 +134,16 @@ if not getattr(sys, 'frozen', False): def pytest_collection_modifyitems(config, items): """Apply @qtwebengine_* markers; skip unittests with QUTE_BDD_WEBENGINE.""" + if config.webengine: + qtwebkit_ng_used = False + else: + try: + from PyQt5.QtWebKit import qWebKitVersion + except ImportError: + qtwebkit_ng_used = False + else: + qtwebkit_ng_used = qtutils.is_qtwebkit_ng(qWebKitVersion()) + markers = [ ('qtwebengine_todo', 'QtWebEngine TODO', pytest.mark.xfail, config.webengine), @@ -139,6 +151,10 @@ def pytest_collection_modifyitems(config, items): config.webengine), ('qtwebkit_skip', 'Skipped with QtWebKit', pytest.mark.skipif, not config.webengine), + ('qtwebkit_ng_xfail', 'Failing with QtWebKit-NG', pytest.mark.xfail, + qtwebkit_ng_used), + ('qtwebkit_ng_skip', 'Skipped with QtWebKit-NG', pytest.mark.skipif, + qtwebkit_ng_used), ('qtwebengine_flaky', 'Flaky with QtWebEngine', pytest.mark.skipif, config.webengine), ('qtwebengine_osx_xfail', 'Fails on OS X with QtWebEngine', diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 3ee531f42..7bfb66921 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -560,7 +560,7 @@ Feature: Downloading things from a website. And I run :download foo! Then the error "Invalid URL" should be shown - @qtwebengine_todo: pdfjs is not implemented yet + @qtwebengine_todo: pdfjs is not implemented yet @qtwebkit_ng_xfail: https://github.com/annulen/webkit/issues/428 Scenario: Downloading via pdfjs Given pdfjs is available When I set storage -> prompt-download-directory to false diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 7e9c55c99..3c35ad03f 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -69,7 +69,7 @@ Feature: Page history ## Bugs - @qtwebengine_skip + @qtwebengine_skip @qtwebkit_ng_skip Scenario: Opening a valid URL which turns out invalid When I set general -> auto-search to true And I run :open http://foo%40bar@baz diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index 19e103476..13ab0d96a 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -16,6 +16,7 @@ Feature: Javascript stuff And I run :click-element id close-normal Then "Focus object changed: *" should be logged + @qtwebkit_ng_skip Scenario: Opening/closing a modal window via JS When I open data/javascript/window_open.html And I run :tab-only diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 168ea98ca..7fbc77c89 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -318,7 +318,7 @@ Feature: Various utility commands. And I open data/misc/test.pdf Then "Download test.pdf finished" should be logged - @qtwebengine_skip: pdfjs is not implemented yet + @qtwebengine_skip: pdfjs is not implemented yet @qtwebkit_ng_xfail: https://github.com/annulen/webkit/issues/428 Scenario: Downloading a pdf via pdf.js button (issue 1214) Given pdfjs is available # WORKAROUND to prevent the "Painter ended with 2 saved states" warning @@ -523,7 +523,7 @@ Feature: Various utility commands. ## https://github.com/qutebrowser/qutebrowser/issues/1742 - @qtwebengine_todo: private browsing is not implemented yet + @qtwebengine_todo: private browsing is not implemented yet @qtwebkit_ng_xfail: private browsing is not implemented yet Scenario: Private browsing is activated in QtWebKit without restart When I set general -> private-browsing to true And I open data/javascript/localstorage.html From 4ee207b3cb7e53f8a6b655c779c73a8c4603805d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 6 Feb 2017 08:13:43 +0100 Subject: [PATCH 5/6] Remove webkit.tabhistory from check_coverage We can't easily check for both QtWebKit and -NG code in the same run. --- scripts/dev/check_coverage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 987a376ae..bdaac4a14 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -55,8 +55,6 @@ PERFECT_FILES = [ 'qutebrowser/browser/history.py'), ('tests/unit/browser/webkit/test_history.py', 'qutebrowser/browser/webkit/webkithistory.py'), - ('tests/unit/browser/webkit/test_tabhistory.py', - 'qutebrowser/browser/webkit/tabhistory.py'), ('tests/unit/browser/webkit/http/test_http.py', 'qutebrowser/browser/webkit/http.py'), ('tests/unit/browser/webkit/http/test_content_disposition.py', From 165c534f20b1b364ffdbfc32373a15b0be5e4459 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 6 Feb 2017 08:48:58 +0100 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c3aea65e7..0eca7584a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -22,6 +22,7 @@ Added - Userscripts now have a new `$QUTE_COMMANDLINE_TEXT` environment variable, containing the current commandline contents. - New `ripbang` userscript to create a searchengine from a duckduckgo bang +- link:https://github.com/annulen/webkit/wiki[QtWebKit Reloaded] (also called QtWebKit-NG) is now supported. Changed ~~~~~~~