qutebrowser/tests/unit/browser/test_history.py

# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

"""Tests for the global page history."""

import logging

import pytest
from PyQt5.QtCore import QUrl

from qutebrowser.browser import history
from qutebrowser.utils import objreg, urlutils, usertypes
from qutebrowser.commands import cmdexc
from qutebrowser.misc import sql


@pytest.fixture(autouse=True)
def prerequisites(config_stub, fake_save_manager, init_sql, fake_args):
    """Make sure everything is ready to initialize a WebHistory."""
    fake_args.debug_flags = []
    config_stub.data = {'general': {'private-browsing': False}}


class FakeHistoryProgress:

    """Fake for a WebHistoryProgress object."""

    def __init__(self):
        self._started = False
        self._finished = False
        self._value = 0

    def start(self, _text, _maximum):
        self._started = True

    def tick(self):
        self._value += 1

    def finish(self):
        self._finished = True


@pytest.fixture()
def hist(tmpdir):
    return history.WebHistory(progress=FakeHistoryProgress())


class TestSpecialMethods:

    def test_iter(self, hist):
        urlstr = 'http://www.example.com/'
        url = QUrl(urlstr)
        hist.add_url(url, atime=12345)

        assert list(hist) == [(urlstr, '', 12345, False)]

    def test_len(self, hist):
        assert len(hist) == 0

        url = QUrl('http://www.example.com/')
        hist.add_url(url)

        assert len(hist) == 1

    def test_contains(self, hist):
        hist.add_url(QUrl('http://www.example.com/'), title='Title',
                     atime=12345)
        assert 'http://www.example.com/' in hist
        assert 'www.example.com' not in hist
        assert 'Title' not in hist
        assert 12345 not in hist


class TestGetting:

    def test_get_recent(self, hist):
        hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890)
        hist.add_url(QUrl('http://example.com/'), atime=12345)
        assert list(hist.get_recent()) == [
            ('http://www.qutebrowser.org/', '', 67890, False),
            ('http://example.com/', '', 12345, False),
        ]

    def test_entries_between(self, hist):
        hist.add_url(QUrl('http://www.example.com/1'), atime=12345)
        hist.add_url(QUrl('http://www.example.com/2'), atime=12346)
        hist.add_url(QUrl('http://www.example.com/3'), atime=12347)
        hist.add_url(QUrl('http://www.example.com/4'), atime=12348)
        hist.add_url(QUrl('http://www.example.com/5'), atime=12348)
        hist.add_url(QUrl('http://www.example.com/6'), atime=12349)
        hist.add_url(QUrl('http://www.example.com/7'), atime=12350)

        times = [x.atime for x in hist.entries_between(12346, 12349)]
        assert times == [12349, 12348, 12348, 12347]

    def test_entries_before(self, hist):
        hist.add_url(QUrl('http://www.example.com/1'), atime=12346)
        hist.add_url(QUrl('http://www.example.com/2'), atime=12346)
        hist.add_url(QUrl('http://www.example.com/3'), atime=12347)
        hist.add_url(QUrl('http://www.example.com/4'), atime=12348)
        hist.add_url(QUrl('http://www.example.com/5'), atime=12348)
        hist.add_url(QUrl('http://www.example.com/6'), atime=12348)
        hist.add_url(QUrl('http://www.example.com/7'), atime=12349)
        hist.add_url(QUrl('http://www.example.com/8'), atime=12349)

        times = [x.atime for x in
                 hist.entries_before(12348, limit=3, offset=2)]
        assert times == [12348, 12347, 12346]


class TestDelete:

    def test_clear(self, qtbot, tmpdir, hist, mocker):
        hist.add_url(QUrl('http://example.com/'))
        hist.add_url(QUrl('http://www.qutebrowser.org/'))

        m = mocker.patch('qutebrowser.browser.history.message.confirm_async',
                         new=mocker.Mock, spec=[])
        hist.clear()
        assert m.called

    def test_clear_force(self, qtbot, tmpdir, hist):
        hist.add_url(QUrl('http://example.com/'))
        hist.add_url(QUrl('http://www.qutebrowser.org/'))
        hist.clear(force=True)
        assert not len(hist)
        assert not len(hist.completion)

    @pytest.mark.parametrize('raw, escaped', [
        ('http://example.com/1', 'http://example.com/1'),
        ('http://example.com/1 2', 'http://example.com/1%202'),
    ])
    def test_delete_url(self, hist, raw, escaped):
        hist.add_url(QUrl('http://example.com/'), atime=0)
        hist.add_url(QUrl(escaped), atime=0)
        hist.add_url(QUrl('http://example.com/2'), atime=0)

        before = set(hist)
        completion_before = set(hist.completion)

        hist.delete_url(QUrl(raw))

        diff = before.difference(set(hist))
        assert diff == {(escaped, '', 0, False)}

        completion_diff = completion_before.difference(set(hist.completion))
        assert completion_diff == {(raw, '', 0)}


class TestAdd:

    @pytest.fixture()
    def mock_time(self, mocker):
        m = mocker.patch('qutebrowser.browser.history.time')
        m.time.return_value = 12345
        return 12345

    @pytest.mark.parametrize(
        'url, atime, title, redirect, history_url, completion_url', [
            ('http://www.example.com', 12346, 'the title', False,
             'http://www.example.com', 'http://www.example.com'),
            ('http://www.example.com', 12346, 'the title', True,
             'http://www.example.com', None),
            ('http://www.example.com/sp ce', 12346, 'the title', False,
             'http://www.example.com/sp%20ce', 'http://www.example.com/sp ce'),
            ('https://user:pass@example.com', 12346, 'the title', False,
             'https://user@example.com', 'https://user@example.com'),
        ]
    )
    def test_add_url(self, qtbot, hist,
                     url, atime, title, redirect, history_url, completion_url):
        hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect)
        assert list(hist) == [(history_url, title, atime, redirect)]
        if completion_url is None:
            assert not len(hist.completion)
        else:
            assert list(hist.completion) == [(completion_url, title, atime)]

    def test_no_sql_history(self, hist, fake_args):
        fake_args.debug_flags = 'no-sql-history'
        hist.add_url(QUrl('https://www.example.com/'), atime=12346,
                     title='Hello World', redirect=False)
        assert not list(hist)

    def test_invalid(self, qtbot, hist, caplog):
        with caplog.at_level(logging.WARNING):
            hist.add_url(QUrl())
        assert not list(hist)
        assert not list(hist.completion)

    @pytest.mark.parametrize('environmental', [True, False])
    @pytest.mark.parametrize('completion', [True, False])
    def test_error(self, monkeypatch, hist, message_mock, caplog,
                   environmental, completion):
        def raise_error(url, replace=False):
            if environmental:
                raise sql.SqlEnvironmentError("Error message")
            else:
                raise sql.SqlBugError("Error message")

        if completion:
            monkeypatch.setattr(hist.completion, 'insert', raise_error)
        else:
            monkeypatch.setattr(hist, 'insert', raise_error)

        if environmental:
            with caplog.at_level(logging.ERROR):
                hist.add_url(QUrl('https://www.example.org/'))
            msg = message_mock.getmsg(usertypes.MessageLevel.error)
            assert msg.text == "Failed to write history: Error message"
        else:
            with pytest.raises(sql.SqlBugError):
                hist.add_url(QUrl('https://www.example.org/'))

    @pytest.mark.parametrize('level, url, req_url, expected', [
        (logging.DEBUG, 'a.com', 'a.com', [('a.com', 'title', 12345, False)]),
        (logging.DEBUG, 'a.com', 'b.com', [('a.com', 'title', 12345, False),
                                           ('b.com', 'title', 12345, True)]),
        (logging.WARNING, 'a.com', '', [('a.com', 'title', 12345, False)]),
        (logging.WARNING, '', '', []),
        (logging.WARNING, 'data:foo', '', []),
        (logging.WARNING, 'a.com', 'data:foo', []),
    ])
    def test_from_tab(self, hist, caplog, mock_time,
                      level, url, req_url, expected):
        with caplog.at_level(level):
            hist.add_from_tab(QUrl(url), QUrl(req_url), 'title')
        assert set(hist) == set(expected)

    def test_exclude(self, hist, config_stub):
        """Excluded URLs should be in the history but not completion."""
        config_stub.val.completion.web_history.exclude = ['*.example.org']
        url = QUrl('http://www.example.org/')
        hist.add_from_tab(url, url, 'title')
        assert list(hist)
        assert not list(hist.completion)


class TestHistoryInterface:

    @pytest.fixture
    def hist_interface(self, hist):
        # pylint: disable=invalid-name
        QtWebKit = pytest.importorskip('PyQt5.QtWebKit')
        from qutebrowser.browser.webkit import webkithistory
        QWebHistoryInterface = QtWebKit.QWebHistoryInterface
        # pylint: enable=invalid-name
        hist.add_url(url=QUrl('http://www.example.com/'), title='example')
        interface = webkithistory.WebHistoryInterface(hist)
        QWebHistoryInterface.setDefaultInterface(interface)
        yield
        QWebHistoryInterface.setDefaultInterface(None)

    def test_history_interface(self, qtbot, webview, hist_interface):
        html = b"<a href='about:blank'>foo</a>"
        url = urlutils.data_url('text/html', html)
        with qtbot.waitSignal(webview.loadFinished):
            webview.load(url)


class TestInit:

    @pytest.fixture
    def cleanup_init(self):
        # prevent test_init from leaking state
        yield
        hist = objreg.get('web-history', None)
        if hist is not None:
            hist.setParent(None)
            objreg.delete('web-history')
        try:
            from PyQt5.QtWebKit import QWebHistoryInterface
            QWebHistoryInterface.setDefaultInterface(None)
        except ImportError:
            pass

    @pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine,
                                         usertypes.Backend.QtWebKit])
    def test_init(self, backend, qapp, tmpdir, monkeypatch, cleanup_init):
        if backend == usertypes.Backend.QtWebKit:
            pytest.importorskip('PyQt5.QtWebKitWidgets')
        else:
            assert backend == usertypes.Backend.QtWebEngine

        monkeypatch.setattr(history.objects, 'backend', backend)
        history.init(qapp)
        hist = objreg.get('web-history')
        assert hist.parent() is qapp

        try:
            from PyQt5.QtWebKit import QWebHistoryInterface
        except ImportError:
            QWebHistoryInterface = None

        if backend == usertypes.Backend.QtWebKit:
            default_interface = QWebHistoryInterface.defaultInterface()
            assert default_interface._history is hist
        else:
            assert backend == usertypes.Backend.QtWebEngine
            if QWebHistoryInterface is None:
                default_interface = None
            else:
                default_interface = QWebHistoryInterface.defaultInterface()
            # For this to work, nothing can ever have called
            # setDefaultInterface before (so we need to test webengine before
            # webkit)
            assert default_interface is None


class TestDump:

    def test_debug_dump_history(self, hist, tmpdir):
        hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345)
        hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346)
        hist.add_url(QUrl('http://example.com/3'), title="Title3", atime=12347)
        hist.add_url(QUrl('http://example.com/4'), title="Title4", atime=12348,
                     redirect=True)
        histfile = tmpdir / 'history'
        hist.debug_dump_history(str(histfile))
        expected = ['12345 http://example.com/1 Title1',
                    '12346 http://example.com/2 Title2',
                    '12347 http://example.com/3 Title3',
                    '12348-r http://example.com/4 Title4']
        assert histfile.read() == '\n'.join(expected)

    def test_nonexistent(self, hist, tmpdir):
        histfile = tmpdir / 'nonexistent' / 'history'
        with pytest.raises(cmdexc.CommandError):
            hist.debug_dump_history(str(histfile))


class TestRebuild:

    def test_delete(self, hist):
        hist.insert({'url': 'example.com/1', 'title': 'example1',
                     'redirect': False, 'atime': 1})
        hist.insert({'url': 'example.com/1', 'title': 'example1',
                     'redirect': False, 'atime': 2})
        hist.insert({'url': 'example.com/2%203', 'title': 'example2',
                     'redirect': False, 'atime': 3})
        hist.insert({'url': 'example.com/3', 'title': 'example3',
                     'redirect': True, 'atime': 4})
        hist.insert({'url': 'example.com/2 3', 'title': 'example2',
                     'redirect': False, 'atime': 5})
        hist.completion.delete_all()

        hist2 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist2.completion) == [
            ('example.com/1', 'example1', 2),
            ('example.com/2 3', 'example2', 5),
        ]

    def test_no_rebuild(self, hist):
        """Ensure that completion is not regenerated unless empty."""
        hist.add_url(QUrl('example.com/1'), redirect=False, atime=1)
        hist.add_url(QUrl('example.com/2'), redirect=False, atime=2)
        hist.completion.delete('url', 'example.com/2')

        hist2 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist2.completion) == [('example.com/1', '', 1)]

    def test_user_version(self, hist, monkeypatch):
        """Ensure that completion is regenerated if user_version changes."""
        hist.add_url(QUrl('example.com/1'), redirect=False, atime=1)
        hist.add_url(QUrl('example.com/2'), redirect=False, atime=2)
        hist.completion.delete('url', 'example.com/2')

        hist2 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist2.completion) == [('example.com/1', '', 1)]

        monkeypatch.setattr(history, '_USER_VERSION',
                            history._USER_VERSION + 1)
        hist3 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist3.completion) == [
            ('example.com/1', '', 1),
            ('example.com/2', '', 2),
        ]

    def test_force_rebuild(self, hist):
        """Ensure that completion is regenerated if we force a rebuild."""
        hist.add_url(QUrl('example.com/1'), redirect=False, atime=1)
        hist.add_url(QUrl('example.com/2'), redirect=False, atime=2)
        hist.completion.delete('url', 'example.com/2')

        hist2 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist2.completion) == [('example.com/1', '', 1)]
        hist2.metainfo['force_rebuild'] = True

        hist3 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist3.completion) == [
            ('example.com/1', '', 1),
            ('example.com/2', '', 2),
        ]
        assert not hist3.metainfo['force_rebuild']

    def test_exclude(self, config_stub, hist):
        """Ensure that patterns in completion.web_history.exclude are ignored.

        This setting should only be used for the completion.
        """
        config_stub.val.completion.web_history.exclude = ['*.example.org']
        assert hist.metainfo['force_rebuild']

        hist.add_url(QUrl('http://example.com'), redirect=False, atime=1)
        hist.add_url(QUrl('http://example.org'), redirect=False, atime=2)

        hist2 = history.WebHistory(progress=FakeHistoryProgress())
        assert list(hist2.completion) == [('http://example.com', '', 1)]

    def test_unrelated_config_change(self, config_stub, hist):
        config_stub.val.history_gap_interval = 1234
        assert not hist.metainfo['force_rebuild']

    @pytest.mark.parametrize('patch_threshold', [True, False])
    def test_progress(self, hist, config_stub, monkeypatch, patch_threshold):
        hist.add_url(QUrl('example.com/1'), redirect=False, atime=1)
        hist.add_url(QUrl('example.com/2'), redirect=False, atime=2)
        hist.metainfo['force_rebuild'] = True

        if patch_threshold:
            monkeypatch.setattr(history.WebHistory, '_PROGRESS_THRESHOLD', 1)

        progress = FakeHistoryProgress()
        history.WebHistory(progress=progress)
        assert progress._value == 2
        assert progress._finished
        assert progress._started == patch_threshold


class TestCompletionMetaInfo:

    @pytest.fixture
    def metainfo(self):
        return history.CompletionMetaInfo()

    def test_contains_keyerror(self, metainfo):
        with pytest.raises(KeyError):
            'does_not_exist' in metainfo  # pylint: disable=pointless-statement

    def test_getitem_keyerror(self, metainfo):
        with pytest.raises(KeyError):
            metainfo['does_not_exist']  # pylint: disable=pointless-statement

    def test_setitem_keyerror(self, metainfo):
        with pytest.raises(KeyError):
            metainfo['does_not_exist'] = 42

    def test_contains(self, metainfo):
        assert 'force_rebuild' in metainfo

    def test_modify(self, metainfo):
        assert not metainfo['force_rebuild']
        metainfo['force_rebuild'] = True
        assert metainfo['force_rebuild']


class TestHistoryProgress:

    @pytest.fixture
    def progress(self):
        return history.HistoryProgress()

    def test_no_start(self, progress):
        """Test calling tick/finish without start."""
        progress.tick()
        progress.finish()
        assert progress._progress is None
        assert progress._value == 1

    def test_gui(self, qtbot, progress):
        progress.start("Hello World", 42)
        dialog = progress._progress
        qtbot.add_widget(dialog)
        progress.tick()

        assert dialog.isVisible()
        assert dialog.labelText() == "Hello World"
        assert dialog.minimum() == 0
        assert dialog.maximum() == 42
        assert dialog.value() == 1
        assert dialog.minimumDuration() == 500

        progress.finish()
        assert not dialog.isVisible()