Merge branch 'webkit-ng-history'

.travis.yml

@@ -15,6 +15,9 @@ matrix:
     - os: linux
       env: DOCKER=archlinux QUTE_BDD_WEBENGINE=true
       services: docker
+    - os: linux
+      env: DOCKER=archlinux-ng
+      services: docker
     - os: linux
       env: DOCKER=ubuntu-xenial
       services: docker

CHANGELOG.asciidoc

@@ -22,6 +22,7 @@ Added
 
 - Userscripts now have a new `$QUTE_COMMANDLINE_TEXT` environment variable, containing the current commandline contents.
 - New `ripbang` userscript to create a searchengine from a duckduckgo bang
+- link:https://github.com/annulen/webkit/wiki[QtWebKit Reloaded] (also called QtWebKit-NG) is now supported.
 
 Changed
 ~~~~~~~

pytest.ini

@@ -17,6 +17,8 @@ markers =
     qtwebengine_todo: Features still missing with QtWebEngine
     qtwebengine_skip: Tests not applicable with QtWebEngine
     qtwebkit_skip: Tests not applicable with QtWebKit
+    qtwebkit_ng_xfail: Tests failing with QtWebKit-NG
+    qtwebkit_ng_skip: Tests skipped with QtWebKit-NG
     qtwebengine_flaky: Tests which are flaky (and currently skipped) with QtWebEngine
     qtwebengine_osx_xfail: Tests which fail on OS X with QtWebEngine
     js_prompt: Tests needing to display a javascript prompt

qutebrowser/browser/webkit/tabhistory.py

@@ -21,21 +21,65 @@
 
 
 from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl
+from PyQt5.QtWebKit import qWebKitVersion
 
 from qutebrowser.utils import qtutils
 
 
-HISTORY_STREAM_VERSION = 2
-BACK_FORWARD_TREE_VERSION = 2
-
-
 def _encode_url(url):
     """Encode a QUrl suitable to pass to QWebHistory."""
     data = bytes(QUrl.toPercentEncoding(url.toString(), b':/#?&+=@%*'))
     return data.decode('ascii')
 
 
-def _serialize_item(i, item, stream):
+def _serialize_ng(items, current_idx, stream):
+    # {'currentItemIndex': 0,
+    #  'history': [{'children': [],
+    #               'documentSequenceNumber': 1485030525573123,
+    #               'documentState': [],
+    #               'formContentType': '',
+    #               'itemSequenceNumber': 1485030525573122,
+    #               'originalURLString': 'about:blank',
+    #               'pageScaleFactor': 0.0,
+    #               'referrer': '',
+    #               'scrollPosition': {'x': 0, 'y': 0},
+    #               'target': '',
+    #               'title': '',
+    #               'urlString': 'about:blank'}]}
+    data = {'currentItemIndex': current_idx, 'history': []}
+    for item in items:
+        data['history'].append(_serialize_item_ng(item))
+
+    stream.writeInt(3)  # history stream version
+    stream.writeQVariantMap(data)
+
+
+def _serialize_item_ng(item):
+    data = {
+        'originalURLString': item.original_url.toString(QUrl.FullyEncoded),
+        'scrollPosition': {'x': 0, 'y': 0},
+        'title': item.title,
+        'urlString': item.url.toString(QUrl.FullyEncoded),
+    }
+    try:
+        data['scrollPosition']['x'] = item.user_data['scroll-pos'].x()
+        data['scrollPosition']['y'] = item.user_data['scroll-pos'].y()
+    except (KeyError, TypeError):
+        pass
+    return data
+
+
+def _serialize_old(items, current_idx, stream):
+    ### Source/WebKit/qt/Api/qwebhistory.cpp operator<<
+    stream.writeInt(2)  # history stream version
+    stream.writeInt(len(items))
+    stream.writeInt(current_idx)
+
+    for i, item in enumerate(items):
+        _serialize_item_old(i, item, stream)
+
+
+def _serialize_item_old(i, item, stream):
     """Serialize a single WebHistoryItem into a QDataStream.
 
     Args:
@@ -53,7 +97,7 @@ def _serialize_item(i, item, stream):
 
     ### Source/WebCore/history/HistoryItem.cpp decodeBackForwardTree
     ## backForwardTreeEncodingVersion
-    stream.writeUInt32(BACK_FORWARD_TREE_VERSION)
+    stream.writeUInt32(2)
     ## size (recursion stack)
     stream.writeUInt64(0)
     ## node->m_documentSequenceNumber
@@ -137,14 +181,12 @@ def serialize(items):
     else:
         current_idx = 0
 
-    ### Source/WebKit/qt/Api/qwebhistory.cpp operator<<
+    if qtutils.is_qtwebkit_ng(qWebKitVersion()):
-    stream.writeInt(HISTORY_STREAM_VERSION)
+        _serialize_ng(items, current_idx, stream)
-    stream.writeInt(len(items))
+    else:
-    stream.writeInt(current_idx)
+        _serialize_old(items, current_idx, stream)
 
-    for i, item in enumerate(items):
+    user_data += [item.user_data for item in items]
-        _serialize_item(i, item, stream)
-        user_data.append(item.user_data)
 
     stream.device().reset()
     qtutils.check_qdatastream(stream)

qutebrowser/utils/qtutils.py

@@ -91,6 +91,16 @@ def version_check(version, op=operator.ge):
               pkg_resources.parse_version(version))
 
 
+def is_qtwebkit_ng(version):
+    """Check if the given version is  QtWebKit-NG.
+
+    This is typically used as is_webkit_ng(qWebKitVersion) but we don't want to
+    have QtWebKit imports in here.
+    """
+    return (pkg_resources.parse_version(version) >
+            pkg_resources.parse_version('538.1'))
+
+
 def check_overflow(arg, ctype, fatal=True):
     """Check if the given argument is in bounds for the given type.
 

scripts/dev/check_coverage.py

@@ -55,8 +55,6 @@ PERFECT_FILES = [
         'qutebrowser/browser/history.py'),
     ('tests/unit/browser/webkit/test_history.py',
         'qutebrowser/browser/webkit/webkithistory.py'),
-    ('tests/unit/browser/webkit/test_tabhistory.py',
-        'qutebrowser/browser/webkit/tabhistory.py'),
     ('tests/unit/browser/webkit/http/test_http.py',
         'qutebrowser/browser/webkit/http.py'),
     ('tests/unit/browser/webkit/http/test_content_disposition.py',

tests/end2end/conftest.py

@@ -106,7 +106,9 @@ def _get_backend_tag(tag):
     pytest_marks = {
         'qtwebengine_todo': pytest.mark.qtwebengine_todo,
         'qtwebengine_skip': pytest.mark.qtwebengine_skip,
-        'qtwebkit_skip': pytest.mark.qtwebkit_skip
+        'qtwebkit_skip': pytest.mark.qtwebkit_skip,
+        'qtwebkit_ng_xfail': pytest.mark.qtwebkit_ng_xfail,
+        'qtwebkit_ng_skip': pytest.mark.qtwebkit_ng_skip,
     }
     if not any(tag.startswith(t + ':') for t in pytest_marks):
         return None
@@ -132,6 +134,16 @@ if not getattr(sys, 'frozen', False):
 
 def pytest_collection_modifyitems(config, items):
     """Apply @qtwebengine_* markers; skip unittests with QUTE_BDD_WEBENGINE."""
+    if config.webengine:
+        qtwebkit_ng_used = False
+    else:
+        try:
+            from PyQt5.QtWebKit import qWebKitVersion
+        except ImportError:
+            qtwebkit_ng_used = False
+        else:
+            qtwebkit_ng_used = qtutils.is_qtwebkit_ng(qWebKitVersion())
+
     markers = [
         ('qtwebengine_todo', 'QtWebEngine TODO', pytest.mark.xfail,
          config.webengine),
@@ -139,6 +151,10 @@ def pytest_collection_modifyitems(config, items):
          config.webengine),
         ('qtwebkit_skip', 'Skipped with QtWebKit', pytest.mark.skipif,
          not config.webengine),
+        ('qtwebkit_ng_xfail', 'Failing with QtWebKit-NG', pytest.mark.xfail,
+         qtwebkit_ng_used),
+        ('qtwebkit_ng_skip', 'Skipped with QtWebKit-NG', pytest.mark.skipif,
+         qtwebkit_ng_used),
         ('qtwebengine_flaky', 'Flaky with QtWebEngine', pytest.mark.skipif,
          config.webengine),
         ('qtwebengine_osx_xfail', 'Fails on OS X with QtWebEngine',

tests/end2end/features/downloads.feature

@@ -560,7 +560,7 @@ Feature: Downloading things from a website.
         And I run :download foo!
         Then the error "Invalid URL" should be shown
 
-    @qtwebengine_todo: pdfjs is not implemented yet
+    @qtwebengine_todo: pdfjs is not implemented yet @qtwebkit_ng_xfail: https://github.com/annulen/webkit/issues/428
     Scenario: Downloading via pdfjs
         Given pdfjs is available
         When I set storage -> prompt-download-directory to false

tests/end2end/features/history.feature

@@ -69,7 +69,7 @@ Feature: Page history
 
     ## Bugs
 
-    @qtwebengine_skip
+    @qtwebengine_skip @qtwebkit_ng_skip
     Scenario: Opening a valid URL which turns out invalid
         When I set general -> auto-search to true
         And I run :open http://foo%40bar@baz

tests/end2end/features/javascript.feature

@@ -16,6 +16,7 @@ Feature: Javascript stuff
         And I run :click-element id close-normal
         Then "Focus object changed: *" should be logged
 
+    @qtwebkit_ng_skip
     Scenario: Opening/closing a modal window via JS
         When I open data/javascript/window_open.html
         And I run :tab-only

tests/end2end/features/misc.feature

@@ -318,7 +318,7 @@ Feature: Various utility commands.
         And I open data/misc/test.pdf
         Then "Download test.pdf finished" should be logged
 
-    @qtwebengine_skip: pdfjs is not implemented yet
+    @qtwebengine_skip: pdfjs is not implemented yet @qtwebkit_ng_xfail: https://github.com/annulen/webkit/issues/428
     Scenario: Downloading a pdf via pdf.js button (issue 1214)
         Given pdfjs is available
         # WORKAROUND to prevent the "Painter ended with 2 saved states" warning
@@ -523,7 +523,7 @@ Feature: Various utility commands.
 
     ## https://github.com/qutebrowser/qutebrowser/issues/1742
 
-    @qtwebengine_todo: private browsing is not implemented yet
+    @qtwebengine_todo: private browsing is not implemented yet @qtwebkit_ng_xfail: private browsing is not implemented yet
     Scenario: Private browsing is activated in QtWebKit without restart
         When I set general -> private-browsing to true
         And I open data/javascript/localstorage.html

tests/unit/utils/test_qtutils.py

@@ -64,6 +64,15 @@ def test_version_check(monkeypatch, qversion, version, op, expected):
     assert qtutils.version_check(version, op) == expected
 
 
+@pytest.mark.parametrize('version, ng', [
+    ('537.21', False),  # QtWebKit 5.1
+    ('538.1', False),   # Qt 5.8
+    ('602.1', True)     # QtWebKit-NG TP5
+])
+def test_is_qtwebkit_ng(version, ng):
+    assert qtutils.is_qtwebkit_ng(version) == ng
+
+
 class TestCheckOverflow:
 
     """Test check_overflow."""