From fbe5386e5698805c6583724b39fe0b62036aaa00 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sat, 19 Sep 2015 20:06:15 +0200
Subject: [PATCH 01/66] Initial version of website downloader

Saving websites as MHTML via :download-whole

Still needs some cleanup and a "ask for save path".
---
 qutebrowser/browser/commands.py  |  12 +-
 qutebrowser/browser/downloads.py |  14 ++
 qutebrowser/misc/mhtml.py        | 211 +++++++++++++++++++++++++++++++
 3 files changed, 236 insertions(+), 1 deletion(-)
 create mode 100644 qutebrowser/misc/mhtml.py

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index ae4b1da39..10f1d0233 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -42,7 +42,7 @@ from qutebrowser.keyinput import modeman
 from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
                                objreg, utils)
 from qutebrowser.utils.usertypes import KeyMode
-from qutebrowser.misc import editor, guiprocess
+from qutebrowser.misc import editor, guiprocess, mhtml
 
 
 class CommandDispatcher:
@@ -1157,6 +1157,16 @@ class CommandDispatcher:
             page = self._current_widget().page()
             download_manager.get(self._current_url(), page=page)
 
+    @cmdutils.register(instance='command-dispatcher', scope='window')
+    def download_whole(self, dest):
+        """Download the current page as a MHTML file, including all assets
+        (e.g. images)
+
+        Args:
+            dest: The file path to write the download to.
+        """
+        mhtml.start_download(dest)
+
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        deprecated="Use :download instead.")
     def download_page(self):
diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index 70760acb5..568e7bf67 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -201,6 +201,7 @@ class DownloadItem(QObject):
         fileobj: The file object to download the file to.
         reply: The QNetworkReply associated with this download.
         retry_info: A RetryInfo instance.
+        raw_headers: The headers sent by the server.
         _filename: The filename of the download.
         _redirects: How many time we were redirected already.
         _buffer: A BytesIO object to buffer incoming data until we know the
@@ -255,6 +256,7 @@ class DownloadItem(QObject):
         self._filename = None
         self.init_reply(reply)
         self._win_id = win_id
+        self.raw_headers = {}
 
     def __repr__(self):
         return utils.get_repr(self, basename=self.basename)
@@ -354,6 +356,7 @@ class DownloadItem(QObject):
         reply.finished.connect(self.on_reply_finished)
         reply.error.connect(self.on_reply_error)
         reply.readyRead.connect(self.on_ready_read)
+        reply.metaDataChanged.connect(self.on_meta_data_change)
         self.retry_info = RetryInfo(request=reply.request(),
                                     manager=reply.manager())
         if not self.fileobj:
@@ -582,6 +585,9 @@ class DownloadItem(QObject):
         if code == QNetworkReply.OperationCanceledError:
             return
         else:
+            if self.reply is None:
+                log.downloads.debug("QNetworkReply disappeared %s", self)
+                return
             self._die(self.reply.errorString())
 
     @pyqtSlot()
@@ -593,6 +599,14 @@ class DownloadItem(QObject):
         if data is not None:
             self._buffer.write(data)
 
+    @pyqtSlot()
+    def on_meta_data_change(self):
+        if self.reply is None:
+            return
+        self.raw_headers = {}
+        for key, value in self.reply.rawHeaderPairs():
+            self.raw_headers[bytes(key)] = bytes(value)
+
     def _handle_redirect(self):
         """Handle a HTTP redirect.
 
diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
new file mode 100644
index 000000000..3afc32c02
--- /dev/null
+++ b/qutebrowser/misc/mhtml.py
@@ -0,0 +1,211 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015 Daniel Schadt
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Utils for writing a MHTML file."""
+
+import functools
+import io
+
+from collections import namedtuple
+from base64 import b64encode
+from urllib.parse import urljoin
+
+from PyQt5.QtCore import QUrl
+from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
+
+from qutebrowser.utils import log, objreg
+
+
+_File = namedtuple("_File",
+                   "content content_type content_location transfer_encoding")
+
+
+E_NONE = (None, lambda x: x)
+"""No transfer encoding, copy the bytes from input to output"""
+
+E_BASE64 = ("BASE64", b64encode)
+"""Encode the file using base64 encoding"""
+
+
+class MHTMLWriter(object):
+    """A class for aggregating multiple files and outputting them to a MHTML
+    file."""
+
+    BOUNDARY = b"qute-mhtml"
+
+    def __init__(self, root_content=None, content_location=None,
+                 content_type=None):
+        self.root_content = root_content
+        self.content_location = content_location
+        self.content_type = None
+
+        self._files = {}
+
+    def add_file(self, location, content, content_type=None,
+                 transfer_encoding=E_BASE64):
+        """Add a file to the given MHTML collection.
+
+        Args:
+            location: The original location (URL) of the file.
+            content: The binary content of the file.
+            content_type: The MIME-type of the content (if available)
+            transfer_encoding: The transfer encoding to use for this file.
+        """
+        self._files[location] = _File(
+            content=content, content_type=content_type,
+            content_location=location, transfer_encoding=transfer_encoding,
+        )
+
+    def remove_file(self, location):
+        """Remove a file.
+
+        Args:
+            location: The URL that identifies the file.
+        """
+        del self._files[location]
+
+    def write_to(self, fp):
+        """Output the MHTML file to the given file-like object
+
+        Args:
+            fp: The file-object, openend in "wb" mode.
+        """
+        self._output_header(fp)
+        self._output_root_file(fp)
+        for file_data in self._files.values():
+            self._output_file(fp, file_data)
+
+    def _output_header(self, fp):
+        if self.content_location is None:
+            raise ValueError("content_location must be set")
+        if self.content_type is None:
+            raise ValueError("content_type must be set for the root document")
+
+        fp.write(b"Content-Location: ")
+        fp.write(self.content_location.encode("utf-8"))
+        fp.write(b'\nContent-Type: multipart/related;boundary="')
+        fp.write(self.BOUNDARY)
+        fp.write(b'";type="')
+        fp.write(self.content_type.encode("utf-8"))
+        fp.write(b'"\n\n')
+
+    def _output_root_file(self, fp):
+        root_file = _File(
+            content=self.root_content, content_type=self.content_type,
+            content_location=self.content_location, transfer_encoding=E_BASE64
+        )
+        self._output_file(fp, root_file)
+
+    def _output_file(self, fp, file_struct):
+        fp.write(b"--")
+        fp.write(self.BOUNDARY)
+        fp.write(b"\nContent-Location: ")
+        fp.write(file_struct.content_location.encode("utf-8"))
+        if file_struct.content_type is not None:
+            fp.write(b"\nContent-Type: ")
+            fp.write(file_struct.content_type.encode("utf-8"))
+        encoding_name, encoding_func = file_struct.transfer_encoding
+        if encoding_name:
+            fp.write(b"\nContent-Transfer-Encoding: ")
+            fp.write(encoding_name.encode("utf-8"))
+        fp.write(b"\n\n")
+        fp.write(encoding_func(file_struct.content))
+        fp.write(b"\n\n")
+
+
+def start_download(dest):
+    """Start downloading the current page and all assets to a MHTML file.
+
+    Args:
+        dest: The filename where the resulting file should be saved.
+    """
+    download_manager = objreg.get("download-manager", scope="window",
+                                  window="current")
+    web_view = objreg.get("webview", scope="tab", tab="current")
+    web_url_str = web_view.url().toString()
+    web_frame = web_view.page().mainFrame()
+
+    writer = MHTMLWriter()
+    writer.root_content = web_frame.toHtml().encode("utf-8")
+    writer.content_location = web_url_str
+    # I've found no way of getting the content type of a QWebView, but since
+    # we're using .toHtml, it's probably safe to say that the content-type is
+    # HTML
+    writer.content_type = "text/html"
+    # Currently only downloading <link> (stylesheets), <script> (javascript) and
+    # <img> (image) elements.
+    elements = (web_frame.findAllElements("link") +
+                web_frame.findAllElements("script") +
+                web_frame.findAllElements("img"))
+
+    loaded_urls = set()
+    pending_downloads = set()
+
+    # Callback for when a single asset is downloaded
+    # closes over the local variables
+    def finished(name, item):
+        pending_downloads.remove(item)
+        mime = item.raw_headers.get(b"Content-Type", b"")
+        mime = mime.decode("ascii", "ignore")
+        writer.add_file(name, item.fileobj.getvalue(), mime)
+        if pending_downloads:
+            return
+        finish_file()
+
+    def error(item, *args):
+        pending_downloads.remove(item)
+        if pending_downloads:
+            return
+        finish_file()
+
+    def finish_file():
+        # If we get here, all assets are downloaded and we're ready to finis
+        # the file
+        log.misc.debug("All assets downloaded, ready to finish off!")
+        with open(dest, "wb") as file_output:
+            writer.write_to(file_output)
+
+
+    for element in elements:
+        element_url = element.attribute("src")
+        if not element_url:
+            element_url = element.attribute("href")
+        if not element_url:
+            # Might be a local <script> tag or something else
+            continue
+        absolute_url_str = urljoin(web_url_str, element_url)
+        name = absolute_url_str if element_url.startswith("//") else element_url
+        # Prevent loading an asset twice
+        if absolute_url_str in loaded_urls:
+            continue
+        loaded_urls.add(absolute_url_str)
+
+        log.misc.debug("asset at %s", absolute_url_str)
+        absolute_url = QUrl(absolute_url_str)
+
+        fileobj = io.BytesIO()
+        item = download_manager.get(absolute_url, fileobj=fileobj,
+                                    auto_remove=True)
+        pending_downloads.add(item)
+        item.finished.connect(
+            functools.partial(finished, name, item))
+        item.error.connect(
+            functools.partial(error, item))
+        item.cancelled.connect(
+            functools.partial(error, item))

From 930871be01d4893be94a1002782c9931a2be5e60 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 20 Sep 2015 14:30:12 +0200
Subject: [PATCH 02/66] First working version

The files can be opened with qutebrowser

Problems still with Umlauts in the encoded file.
---
 qutebrowser/misc/mhtml.py | 64 +++++++++++++++++++++++++++++----------
 1 file changed, 48 insertions(+), 16 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 3afc32c02..795b102e6 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -20,11 +20,13 @@
 """Utils for writing a MHTML file."""
 
 import functools
+import quopri
 import io
 
 from collections import namedtuple
 from base64 import b64encode
 from urllib.parse import urljoin
+from uuid import uuid4
 
 from PyQt5.QtCore import QUrl
 from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
@@ -36,18 +38,44 @@ _File = namedtuple("_File",
                    "content content_type content_location transfer_encoding")
 
 
+def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
+    """Just like b64encode, except that it breaks long lines.
+
+    Args:
+        maxlen: Maximum length of a line, not including the line separator.
+        linesep: Line separator to use as bytes.
+    """
+    encoded = b64encode(data)
+    result = []
+    for i in range(0, len(encoded), maxlen):
+        result.append(encoded[i:i+maxlen])
+    return linesep.join(result)
+
+def _rn_quopri(data):
+    """Return a quoted-printable representation of data."""
+    orig_funcs = (quopri.b2a_qp, quopri.a2b_qp)
+    # Workaround for quopri mixing \n and \r\n
+    quopri.b2a_qp = quopri.a2b_qp = None
+    encoded = quopri.encodestring(data)
+    quopri.b2a_qp, quopri.a2b_qp = orig_funcs
+    return encoded.replace(b"\n", b"\r\n")
+
+
 E_NONE = (None, lambda x: x)
 """No transfer encoding, copy the bytes from input to output"""
 
-E_BASE64 = ("BASE64", b64encode)
+E_BASE64 = ("base64", _chunked_base64)
 """Encode the file using base64 encoding"""
 
+E_QUOPRI = ("quoted-printable", _rn_quopri)
+"""Encode the file using MIME quoted-printable encoding."""
+
 
 class MHTMLWriter(object):
     """A class for aggregating multiple files and outputting them to a MHTML
     file."""
 
-    BOUNDARY = b"qute-mhtml"
+    BOUNDARY = b"---qute-mhtml-" + str(uuid4()).encode("ascii")
 
     def __init__(self, root_content=None, content_location=None,
                  content_type=None):
@@ -90,6 +118,9 @@ class MHTMLWriter(object):
         self._output_root_file(fp)
         for file_data in self._files.values():
             self._output_file(fp, file_data)
+        fp.write(b"\r\n--")
+        fp.write(self.BOUNDARY)
+        fp.write(b"--")
 
     def _output_header(self, fp):
         if self.content_location is None:
@@ -99,34 +130,34 @@ class MHTMLWriter(object):
 
         fp.write(b"Content-Location: ")
         fp.write(self.content_location.encode("utf-8"))
-        fp.write(b'\nContent-Type: multipart/related;boundary="')
+        fp.write(b'\r\nContent-Type: multipart/related;boundary="')
         fp.write(self.BOUNDARY)
         fp.write(b'";type="')
         fp.write(self.content_type.encode("utf-8"))
-        fp.write(b'"\n\n')
+        fp.write(b'"\r\n\r\n')
 
     def _output_root_file(self, fp):
         root_file = _File(
             content=self.root_content, content_type=self.content_type,
-            content_location=self.content_location, transfer_encoding=E_BASE64
+            content_location=self.content_location, transfer_encoding=E_QUOPRI,
         )
         self._output_file(fp, root_file)
 
     def _output_file(self, fp, file_struct):
         fp.write(b"--")
         fp.write(self.BOUNDARY)
-        fp.write(b"\nContent-Location: ")
+        fp.write(b"\r\nContent-Location: ")
         fp.write(file_struct.content_location.encode("utf-8"))
         if file_struct.content_type is not None:
-            fp.write(b"\nContent-Type: ")
+            fp.write(b"\r\nContent-Type: ")
             fp.write(file_struct.content_type.encode("utf-8"))
         encoding_name, encoding_func = file_struct.transfer_encoding
         if encoding_name:
-            fp.write(b"\nContent-Transfer-Encoding: ")
+            fp.write(b"\r\nContent-Transfer-Encoding: ")
             fp.write(encoding_name.encode("utf-8"))
-        fp.write(b"\n\n")
+        fp.write(b"\r\n\r\n")
         fp.write(encoding_func(file_struct.content))
-        fp.write(b"\n\n")
+        fp.write(b"\r\n\r\n")
 
 
 def start_download(dest):
@@ -163,13 +194,15 @@ def start_download(dest):
         pending_downloads.remove(item)
         mime = item.raw_headers.get(b"Content-Type", b"")
         mime = mime.decode("ascii", "ignore")
-        writer.add_file(name, item.fileobj.getvalue(), mime)
+        encode = E_QUOPRI if mime.startswith("text/") else E_BASE64
+        writer.add_file(name, item.fileobj.getvalue(), mime, encode)
         if pending_downloads:
             return
         finish_file()
 
-    def error(item, *args):
+    def error(name, item, *args):
         pending_downloads.remove(item)
+        writer.add_file(name, b"")
         if pending_downloads:
             return
         finish_file()
@@ -190,7 +223,6 @@ def start_download(dest):
             # Might be a local <script> tag or something else
             continue
         absolute_url_str = urljoin(web_url_str, element_url)
-        name = absolute_url_str if element_url.startswith("//") else element_url
         # Prevent loading an asset twice
         if absolute_url_str in loaded_urls:
             continue
@@ -204,8 +236,8 @@ def start_download(dest):
                                     auto_remove=True)
         pending_downloads.add(item)
         item.finished.connect(
-            functools.partial(finished, name, item))
+            functools.partial(finished, absolute_url_str, item))
         item.error.connect(
-            functools.partial(error, item))
+            functools.partial(error, absolute_url_str, item))
         item.cancelled.connect(
-            functools.partial(error, item))
+            functools.partial(error, absolute_url_str, item))

From 024ae52366cbbbdcbf88d4ab03bbc99e15e562ea Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 20 Sep 2015 16:05:36 +0200
Subject: [PATCH 03/66] Replaced quote-printable with own function

The original one had some inconsistencies that lead to bugs.

The content-type of the root document now also contains the charset.
---
 qutebrowser/misc/mhtml.py | 37 +++++++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 795b102e6..6ac43a8c1 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -53,12 +53,33 @@ def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
 
 def _rn_quopri(data):
     """Return a quoted-printable representation of data."""
-    orig_funcs = (quopri.b2a_qp, quopri.a2b_qp)
-    # Workaround for quopri mixing \n and \r\n
-    quopri.b2a_qp = quopri.a2b_qp = None
-    encoded = quopri.encodestring(data)
-    quopri.b2a_qp, quopri.a2b_qp = orig_funcs
-    return encoded.replace(b"\n", b"\r\n")
+    # See RFC 2045 https://tools.ietf.org/html/rfc2045#section-6.7
+    # The stdlib version in the quopri module has inconsistencies with line
+    # endings and breaks up character escapes over multiple lines, which isn't
+    # understood by qute and leads to jumbled text
+    MAXLEN = 76
+    WHITESPACE = {ord(b"\t"), ord(b" ")}
+    output = []
+    current_line = b""
+    for byte in data:
+        # Literal representation according to (2) and (3)
+        if (ord(b"!") <= byte <= ord(b"<") or
+            ord(b">") <= byte <= ord(b"~") or
+            byte in WHITESPACE):
+            current_line += bytes([byte])
+        else:
+            current_line += b"=" + "{:02X}".format(byte).encode("ascii")
+        if len(current_line) >= MAXLEN:
+            # We need to account for the = character
+            split = [current_line[:MAXLEN-1], current_line[MAXLEN-1:]]
+            quoted_pos = split[0].rfind(b"=")
+            if quoted_pos + 2 >= MAXLEN - 1:
+                split[0], token = split[0][:quoted_pos], split[0][quoted_pos:]
+                split[1] = token + split[1]
+            current_line = split[1]
+            output.append(split[0] + b"=")
+    output.append(current_line)
+    return b"\r\n".join(output)
 
 
 E_NONE = (None, lambda x: x)
@@ -86,7 +107,7 @@ class MHTMLWriter(object):
         self._files = {}
 
     def add_file(self, location, content, content_type=None,
-                 transfer_encoding=E_BASE64):
+                 transfer_encoding=E_QUOPRI):
         """Add a file to the given MHTML collection.
 
         Args:
@@ -178,7 +199,7 @@ def start_download(dest):
     # I've found no way of getting the content type of a QWebView, but since
     # we're using .toHtml, it's probably safe to say that the content-type is
     # HTML
-    writer.content_type = "text/html"
+    writer.content_type = 'text/html; charset="UTF-8"'
     # Currently only downloading <link> (stylesheets), <script> (javascript) and
     # <img> (image) elements.
     elements = (web_frame.findAllElements("link") +

From 49a32f004112fedbe5d8a8ed4b3e2d6bf1e8c33d Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 20 Sep 2015 17:11:28 +0200
Subject: [PATCH 04/66] First round of lint fixes

---
 qutebrowser/browser/commands.py  |  3 +--
 qutebrowser/browser/downloads.py |  1 +
 qutebrowser/misc/mhtml.py        | 38 ++++++++++++++++----------------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 10f1d0233..641e4aaee 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1159,8 +1159,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def download_whole(self, dest):
-        """Download the current page as a MHTML file, including all assets
-        (e.g. images)
+        """Download the current page as a MHTML file, including all assets.
 
         Args:
             dest: The file path to write the download to.
diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index 568e7bf67..c446b6c55 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -601,6 +601,7 @@ class DownloadItem(QObject):
 
     @pyqtSlot()
     def on_meta_data_change(self):
+        """Update the download's metadata."""
         if self.reply is None:
             return
         self.raw_headers = {}
diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 6ac43a8c1..1266fb6a3 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -20,7 +20,6 @@
 """Utils for writing a MHTML file."""
 
 import functools
-import quopri
 import io
 
 from collections import namedtuple
@@ -29,7 +28,6 @@ from urllib.parse import urljoin
 from uuid import uuid4
 
 from PyQt5.QtCore import QUrl
-from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
 
 from qutebrowser.utils import log, objreg
 
@@ -48,32 +46,32 @@ def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
     encoded = b64encode(data)
     result = []
     for i in range(0, len(encoded), maxlen):
-        result.append(encoded[i:i+maxlen])
+        result.append(encoded[i:i + maxlen])
     return linesep.join(result)
 
+
 def _rn_quopri(data):
     """Return a quoted-printable representation of data."""
     # See RFC 2045 https://tools.ietf.org/html/rfc2045#section-6.7
     # The stdlib version in the quopri module has inconsistencies with line
     # endings and breaks up character escapes over multiple lines, which isn't
     # understood by qute and leads to jumbled text
-    MAXLEN = 76
-    WHITESPACE = {ord(b"\t"), ord(b" ")}
+    maxlen = 76
+    whitespace = {ord(b"\t"), ord(b" ")}
     output = []
     current_line = b""
     for byte in data:
         # Literal representation according to (2) and (3)
-        if (ord(b"!") <= byte <= ord(b"<") or
-            ord(b">") <= byte <= ord(b"~") or
-            byte in WHITESPACE):
+        if (ord(b"!") <= byte <= ord(b"<") or ord(b">") <= byte <= ord(b"~")
+                or byte in whitespace):
             current_line += bytes([byte])
         else:
             current_line += b"=" + "{:02X}".format(byte).encode("ascii")
-        if len(current_line) >= MAXLEN:
+        if len(current_line) >= maxlen:
             # We need to account for the = character
-            split = [current_line[:MAXLEN-1], current_line[MAXLEN-1:]]
+            split = [current_line[:maxlen - 1], current_line[maxlen - 1:]]
             quoted_pos = split[0].rfind(b"=")
-            if quoted_pos + 2 >= MAXLEN - 1:
+            if quoted_pos + 2 >= maxlen - 1:
                 split[0], token = split[0][:quoted_pos], split[0][quoted_pos:]
                 split[1] = token + split[1]
             current_line = split[1]
@@ -93,8 +91,8 @@ E_QUOPRI = ("quoted-printable", _rn_quopri)
 
 
 class MHTMLWriter(object):
-    """A class for aggregating multiple files and outputting them to a MHTML
-    file."""
+
+    """A class for outputting multiple files to a MHTML document."""
 
     BOUNDARY = b"---qute-mhtml-" + str(uuid4()).encode("ascii")
 
@@ -102,7 +100,7 @@ class MHTMLWriter(object):
                  content_type=None):
         self.root_content = root_content
         self.content_location = content_location
-        self.content_type = None
+        self.content_type = content_type
 
         self._files = {}
 
@@ -130,7 +128,7 @@ class MHTMLWriter(object):
         del self._files[location]
 
     def write_to(self, fp):
-        """Output the MHTML file to the given file-like object
+        """Output the MHTML file to the given file-like object.
 
         Args:
             fp: The file-object, openend in "wb" mode.
@@ -144,6 +142,7 @@ class MHTMLWriter(object):
         fp.write(b"--")
 
     def _output_header(self, fp):
+        """Output only the header to the given fileobject."""
         if self.content_location is None:
             raise ValueError("content_location must be set")
         if self.content_type is None:
@@ -158,6 +157,7 @@ class MHTMLWriter(object):
         fp.write(b'"\r\n\r\n')
 
     def _output_root_file(self, fp):
+        """Output the root document to the fileobject."""
         root_file = _File(
             content=self.root_content, content_type=self.content_type,
             content_location=self.content_location, transfer_encoding=E_QUOPRI,
@@ -165,6 +165,7 @@ class MHTMLWriter(object):
         self._output_file(fp, root_file)
 
     def _output_file(self, fp, file_struct):
+        """Output the single given file to the fileobject."""
         fp.write(b"--")
         fp.write(self.BOUNDARY)
         fp.write(b"\r\nContent-Location: ")
@@ -200,8 +201,8 @@ def start_download(dest):
     # we're using .toHtml, it's probably safe to say that the content-type is
     # HTML
     writer.content_type = 'text/html; charset="UTF-8"'
-    # Currently only downloading <link> (stylesheets), <script> (javascript) and
-    # <img> (image) elements.
+    # Currently only downloading <link> (stylesheets), <script> (javascript)
+    # and <img> (image) elements.
     elements = (web_frame.findAllElements("link") +
                 web_frame.findAllElements("script") +
                 web_frame.findAllElements("img"))
@@ -221,7 +222,7 @@ def start_download(dest):
             return
         finish_file()
 
-    def error(name, item, *args):
+    def error(name, item, *_args):
         pending_downloads.remove(item)
         writer.add_file(name, b"")
         if pending_downloads:
@@ -235,7 +236,6 @@ def start_download(dest):
         with open(dest, "wb") as file_output:
             writer.write_to(file_output)
 
-
     for element in elements:
         element_url = element.attribute("src")
         if not element_url:

From 111feebf89606df39d5e990aa0c0760bbae5e896 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 20 Sep 2015 17:57:32 +0200
Subject: [PATCH 05/66] Refactor start_download to a class

---
 qutebrowser/misc/mhtml.py | 171 ++++++++++++++++++++++----------------
 1 file changed, 98 insertions(+), 73 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 1266fb6a3..2d8a5e069 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -182,83 +182,108 @@ class MHTMLWriter(object):
         fp.write(b"\r\n\r\n")
 
 
+class _Downloader(object):
+
+    """A class to download whole websites."""
+
+    def __init__(self, web_view, dest):
+        self.web_view = web_view
+        self.dest = dest
+        self.writer = MHTMLWriter()
+        self.loaded_urls = set()
+        self.pending_downloads = set()
+
+    def run(self):
+        """Download and save the page.
+
+        The object must not be reused, you should create a new one if
+        you want to download another page.
+        """
+        download_manager = objreg.get("download-manager", scope="window",
+                                      window="current")
+        web_url_str = self.web_view.url().toString()
+        web_frame = self.web_view.page().mainFrame()
+
+        self.writer.root_content = web_frame.toHtml().encode("utf-8")
+        self.writer.content_location = web_url_str
+        # I've found no way of getting the content type of a QWebView, but
+        # since we're using .toHtml, it's probably safe to say that the
+        # content-type is HTML
+        self.writer.content_type = 'text/html; charset="UTF-8"'
+        # Currently only downloading <link> (stylesheets), <script>
+        # (javascript) and <img> (image) elements.
+        elements = (web_frame.findAllElements("link") +
+                    web_frame.findAllElements("script") +
+                    web_frame.findAllElements("img"))
+
+        for element in elements:
+            element_url = element.attribute("src")
+            if not element_url:
+                element_url = element.attribute("href")
+            if not element_url:
+                # Might be a local <script> tag or something else
+                continue
+            absolute_url = QUrl(urljoin(web_url_str, element_url))
+            # Prevent loading an asset twice
+            if absolute_url in self.loaded_urls:
+                continue
+            self.loaded_urls.add(absolute_url)
+
+            log.misc.debug("asset at %s", absolute_url)
+
+            item = download_manager.get(absolute_url, fileobj=io.BytesIO(),
+                                        auto_remove=True)
+            self.pending_downloads.add(item)
+            item.finished.connect(
+                functools.partial(self.finished, absolute_url, item))
+            item.error.connect(
+                functools.partial(self.error, absolute_url, item))
+            item.cancelled.connect(
+                functools.partial(self.error, absolute_url, item))
+
+    def finished(self, url, item):
+        """Callback when a single asset is downloaded.
+
+        Args:
+            url: The original url of the asset as QUrl.
+            item: The DownloadItem given by the DownloadManager
+        """
+        self.pending_downloads.remove(item)
+        mime = item.raw_headers.get(b"Content-Type", b"")
+        mime = mime.decode("ascii", "ignore")
+        encode = E_QUOPRI if mime.startswith("text/") else E_BASE64
+        self.writer.add_file(url.toString(), item.fileobj.getvalue(), mime,
+                             encode)
+        if self.pending_downloads:
+            return
+        self.finish_file()
+
+    def error(self, url, item, *_args):
+        """Callback when a download error occurs.
+
+        Args:
+            url: The orignal url of the asset as QUrl.
+            item: The DownloadItem given by the DownloadManager.
+        """
+        self.pending_downloads.remove(item)
+        self.writer.add_file(url.toString(), b"")
+        if self.pending_downloads:
+            return
+        self.finish_file()
+
+    def finish_file(self):
+        """Save the file to the filename given in __init__."""
+        log.misc.debug("All assets downloaded, ready to finish off!")
+        with open(self.dest, "wb") as file_output:
+            self.writer.write_to(file_output)
+
+
 def start_download(dest):
     """Start downloading the current page and all assets to a MHTML file.
 
     Args:
         dest: The filename where the resulting file should be saved.
     """
-    download_manager = objreg.get("download-manager", scope="window",
-                                  window="current")
     web_view = objreg.get("webview", scope="tab", tab="current")
-    web_url_str = web_view.url().toString()
-    web_frame = web_view.page().mainFrame()
-
-    writer = MHTMLWriter()
-    writer.root_content = web_frame.toHtml().encode("utf-8")
-    writer.content_location = web_url_str
-    # I've found no way of getting the content type of a QWebView, but since
-    # we're using .toHtml, it's probably safe to say that the content-type is
-    # HTML
-    writer.content_type = 'text/html; charset="UTF-8"'
-    # Currently only downloading <link> (stylesheets), <script> (javascript)
-    # and <img> (image) elements.
-    elements = (web_frame.findAllElements("link") +
-                web_frame.findAllElements("script") +
-                web_frame.findAllElements("img"))
-
-    loaded_urls = set()
-    pending_downloads = set()
-
-    # Callback for when a single asset is downloaded
-    # closes over the local variables
-    def finished(name, item):
-        pending_downloads.remove(item)
-        mime = item.raw_headers.get(b"Content-Type", b"")
-        mime = mime.decode("ascii", "ignore")
-        encode = E_QUOPRI if mime.startswith("text/") else E_BASE64
-        writer.add_file(name, item.fileobj.getvalue(), mime, encode)
-        if pending_downloads:
-            return
-        finish_file()
-
-    def error(name, item, *_args):
-        pending_downloads.remove(item)
-        writer.add_file(name, b"")
-        if pending_downloads:
-            return
-        finish_file()
-
-    def finish_file():
-        # If we get here, all assets are downloaded and we're ready to finis
-        # the file
-        log.misc.debug("All assets downloaded, ready to finish off!")
-        with open(dest, "wb") as file_output:
-            writer.write_to(file_output)
-
-    for element in elements:
-        element_url = element.attribute("src")
-        if not element_url:
-            element_url = element.attribute("href")
-        if not element_url:
-            # Might be a local <script> tag or something else
-            continue
-        absolute_url_str = urljoin(web_url_str, element_url)
-        # Prevent loading an asset twice
-        if absolute_url_str in loaded_urls:
-            continue
-        loaded_urls.add(absolute_url_str)
-
-        log.misc.debug("asset at %s", absolute_url_str)
-        absolute_url = QUrl(absolute_url_str)
-
-        fileobj = io.BytesIO()
-        item = download_manager.get(absolute_url, fileobj=fileobj,
-                                    auto_remove=True)
-        pending_downloads.add(item)
-        item.finished.connect(
-            functools.partial(finished, absolute_url_str, item))
-        item.error.connect(
-            functools.partial(error, absolute_url_str, item))
-        item.cancelled.connect(
-            functools.partial(error, absolute_url_str, item))
+    loader = _Downloader(web_view, dest)
+    loader.run()

From fd7820ea16c78d3b9ba31c4ac5e21e64dd18137a Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 20 Sep 2015 18:34:45 +0200
Subject: [PATCH 06/66] occurs -> occurred

---
 qutebrowser/misc/mhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 2d8a5e069..7a36011a3 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -259,7 +259,7 @@ class _Downloader(object):
         self.finish_file()
 
     def error(self, url, item, *_args):
-        """Callback when a download error occurs.
+        """Callback when a download error occurred.
 
         Args:
             url: The orignal url of the asset as QUrl.

From 679ab65b5ff407034951f2e3c854a86839a3c407 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 20 Sep 2015 20:44:32 +0200
Subject: [PATCH 07/66] Message on finished download

---
 qutebrowser/misc/mhtml.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 7a36011a3..424d405bc 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -29,7 +29,7 @@ from uuid import uuid4
 
 from PyQt5.QtCore import QUrl
 
-from qutebrowser.utils import log, objreg
+from qutebrowser.utils import log, objreg, message
 
 
 _File = namedtuple("_File",
@@ -276,6 +276,7 @@ class _Downloader(object):
         log.misc.debug("All assets downloaded, ready to finish off!")
         with open(self.dest, "wb") as file_output:
             self.writer.write_to(file_output)
+        message.info("current", "Page saved as {}".format(self.dest), True)
 
 
 def start_download(dest):

From 6b086d159d337451d5d2a9363211aa662d0c4733 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Tue, 22 Sep 2015 13:37:03 +0200
Subject: [PATCH 08/66] Ask for filename when none is given

---
 qutebrowser/browser/commands.py | 18 ++++++++++++++++--
 qutebrowser/misc/mhtml.py       |  2 ++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 641e4aaee..6477bf3f4 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -43,6 +43,8 @@ from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
                                objreg, utils)
 from qutebrowser.utils.usertypes import KeyMode
 from qutebrowser.misc import editor, guiprocess, mhtml
+from qutebrowser.browser.downloads import (_path_suggestion as
+                                           download_path_suggestion)
 
 
 class CommandDispatcher:
@@ -1158,13 +1160,25 @@ class CommandDispatcher:
             download_manager.get(self._current_url(), page=page)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def download_whole(self, dest):
+    def download_whole(self, dest=None):
         """Download the current page as a MHTML file, including all assets.
 
         Args:
             dest: The file path to write the download to.
         """
-        mhtml.start_download(dest)
+        if dest is None:
+            suggested_fn = self._current_title() + ".mht"
+            q = usertypes.Question()
+            q.text = "Save page to: "
+            q.mode = usertypes.PromptMode.text
+            q.completed.connect(q.deleteLater)
+            q.default = download_path_suggestion(suggested_fn)
+            q.answered.connect(mhtml.start_download)
+            message_bridge = objreg.get("message-bridge", scope="window",
+                                        window=self._win_id)
+            message_bridge.ask(q, blocking=False)
+        else:
+            mhtml.start_download(dest)
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        deprecated="Use :download instead.")
diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 424d405bc..2353602fc 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -21,6 +21,7 @@
 
 import functools
 import io
+import os
 
 from collections import namedtuple
 from base64 import b64encode
@@ -285,6 +286,7 @@ def start_download(dest):
     Args:
         dest: The filename where the resulting file should be saved.
     """
+    dest = os.path.expanduser(dest)
     web_view = objreg.get("webview", scope="tab", tab="current")
     loader = _Downloader(web_view, dest)
     loader.run()

From 11ed60620ae2630a4654e47e718f622797e8e175 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Tue, 22 Sep 2015 18:48:52 +0200
Subject: [PATCH 09/66] Also load assets referenced in css files

Things like "@import stylesheet.css" and "url(...)".
---
 qutebrowser/misc/mhtml.py | 77 ++++++++++++++++++++++++++++++---------
 1 file changed, 60 insertions(+), 17 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 2353602fc..b45f0f6a9 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -22,6 +22,7 @@
 import functools
 import io
 import os
+import re
 
 from collections import namedtuple
 from base64 import b64encode
@@ -37,6 +38,32 @@ _File = namedtuple("_File",
                    "content content_type content_location transfer_encoding")
 
 
+_CSS_URL_PATTERNS = [re.compile(x) for x in [
+    rb"@import '(?P<url>[^']+)'",
+    rb'@import "(?P<url>[^"]+)"',
+    rb'''url\((?P<url>[^'"][^)]*)\)''',
+    rb'url\("(?P<url>[^"]+)"\)',
+    rb"url\('(?P<url>[^']+)'\)",
+]]
+
+
+def _get_css_imports(data):
+    """Return all assets that are referenced in the given CSS document.
+
+    The returned URLs are relative to the stylesheet's URL.
+
+    Args:
+        data: The content of the stylesheet to scan as bytes.
+    """
+    urls = []
+    for pattern in _CSS_URL_PATTERNS:
+        for match in pattern.finditer(data):
+            url = match.group("url")
+            if url:
+                urls.append(url)
+    return urls
+
+
 def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
     """Just like b64encode, except that it breaks long lines.
 
@@ -191,7 +218,7 @@ class _Downloader(object):
         self.web_view = web_view
         self.dest = dest
         self.writer = MHTMLWriter()
-        self.loaded_urls = set()
+        self.loaded_urls = {web_view.url()}
         self.pending_downloads = set()
 
     def run(self):
@@ -200,8 +227,6 @@ class _Downloader(object):
         The object must not be reused, you should create a new one if
         you want to download another page.
         """
-        download_manager = objreg.get("download-manager", scope="window",
-                                      window="current")
         web_url_str = self.web_view.url().toString()
         web_frame = self.web_view.page().mainFrame()
 
@@ -225,22 +250,32 @@ class _Downloader(object):
                 # Might be a local <script> tag or something else
                 continue
             absolute_url = QUrl(urljoin(web_url_str, element_url))
-            # Prevent loading an asset twice
-            if absolute_url in self.loaded_urls:
-                continue
-            self.loaded_urls.add(absolute_url)
+            self.fetch_url(absolute_url)
 
-            log.misc.debug("asset at %s", absolute_url)
+    def fetch_url(self, url):
+        """Download the given url and add the file to the collection.
 
-            item = download_manager.get(absolute_url, fileobj=io.BytesIO(),
-                                        auto_remove=True)
-            self.pending_downloads.add(item)
-            item.finished.connect(
-                functools.partial(self.finished, absolute_url, item))
-            item.error.connect(
-                functools.partial(self.error, absolute_url, item))
-            item.cancelled.connect(
-                functools.partial(self.error, absolute_url, item))
+        Args:
+            url: The file to download as QUrl.
+        """
+        # Prevent loading an asset twice
+        if url in self.loaded_urls:
+            return
+        self.loaded_urls.add(url)
+
+        log.misc.debug("loading asset at %s", url)
+
+        download_manager = objreg.get("download-manager", scope="window",
+                                      window="current")
+        item = download_manager.get(url, fileobj=io.BytesIO(),
+                                    auto_remove=True)
+        self.pending_downloads.add(item)
+        item.finished.connect(
+            functools.partial(self.finished, url, item))
+        item.error.connect(
+            functools.partial(self.error, url, item))
+        item.cancelled.connect(
+            functools.partial(self.error, url, item))
 
     def finished(self, url, item):
         """Callback when a single asset is downloaded.
@@ -252,6 +287,14 @@ class _Downloader(object):
         self.pending_downloads.remove(item)
         mime = item.raw_headers.get(b"Content-Type", b"")
         mime = mime.decode("ascii", "ignore")
+
+        if mime.lower() == "text/css":
+            import_urls = _get_css_imports(item.fileobj.getvalue())
+            for import_url in import_urls:
+                import_url = import_url.decode("ascii")
+                absolute_url = QUrl(urljoin(url.toString(), import_url))
+                self.fetch_url(absolute_url)
+
         encode = E_QUOPRI if mime.startswith("text/") else E_BASE64
         self.writer.add_file(url.toString(), item.fileobj.getvalue(), mime,
                              encode)

From 5c6b7157204d5bc7c0a43b178471865f38cd06f4 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Tue, 22 Sep 2015 18:52:44 +0200
Subject: [PATCH 10/66] Use QUrl.resolved instead of urlparse.urljoin

---
 qutebrowser/misc/mhtml.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index b45f0f6a9..9c8e66662 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -227,11 +227,11 @@ class _Downloader(object):
         The object must not be reused, you should create a new one if
         you want to download another page.
         """
-        web_url_str = self.web_view.url().toString()
+        web_url = self.web_view.url()
         web_frame = self.web_view.page().mainFrame()
 
         self.writer.root_content = web_frame.toHtml().encode("utf-8")
-        self.writer.content_location = web_url_str
+        self.writer.content_location = web_url.toString()
         # I've found no way of getting the content type of a QWebView, but
         # since we're using .toHtml, it's probably safe to say that the
         # content-type is HTML
@@ -249,7 +249,7 @@ class _Downloader(object):
             if not element_url:
                 # Might be a local <script> tag or something else
                 continue
-            absolute_url = QUrl(urljoin(web_url_str, element_url))
+            absolute_url = web_url.resolved(QUrl(element_url))
             self.fetch_url(absolute_url)
 
     def fetch_url(self, url):
@@ -292,7 +292,7 @@ class _Downloader(object):
             import_urls = _get_css_imports(item.fileobj.getvalue())
             for import_url in import_urls:
                 import_url = import_url.decode("ascii")
-                absolute_url = QUrl(urljoin(url.toString(), import_url))
+                absolute_url = url.resolved(QUrl(import_url))
                 self.fetch_url(absolute_url)
 
         encode = E_QUOPRI if mime.startswith("text/") else E_BASE64

From 991b6d4fc95aa3ebff37ea995533172a351d0031 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Tue, 22 Sep 2015 19:14:16 +0200
Subject: [PATCH 11/66] Remove urljoin import

---
 qutebrowser/misc/mhtml.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 9c8e66662..f2b04f128 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -26,7 +26,6 @@ import re
 
 from collections import namedtuple
 from base64 import b64encode
-from urllib.parse import urljoin
 from uuid import uuid4
 
 from PyQt5.QtCore import QUrl

From 02c1fa123218aba766d60bb6f4a2337df5d3445a Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 23 Sep 2015 14:30:20 +0200
Subject: [PATCH 12/66] Save mhtml if no assets need to be downloaded

---
 qutebrowser/misc/mhtml.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index f2b04f128..e2bb0a4a5 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -119,7 +119,14 @@ E_QUOPRI = ("quoted-printable", _rn_quopri)
 
 class MHTMLWriter(object):
 
-    """A class for outputting multiple files to a MHTML document."""
+    """A class for outputting multiple files to a MHTML document.
+
+    Attributes:
+        root_content: The root content as bytes.
+        content_location: The url of the page as str.
+        content_type: The MIME-type of the root content as str.
+        _files: Mapping of location->_File struct.
+    """
 
     BOUNDARY = b"---qute-mhtml-" + str(uuid4()).encode("ascii")
 
@@ -211,7 +218,16 @@ class MHTMLWriter(object):
 
 class _Downloader(object):
 
-    """A class to download whole websites."""
+    """A class to download whole websites.
+
+    Attributes:
+        web_view: The QWebView which contains the website that will be saved.
+        dest: Destination filename.
+        writer: The MHTMLWriter object which is used to save the page.
+        loaded_urls: A set of QUrls of finished asset downloads.
+        pending_downloads: A set of unfinished DownloadItems.
+        _finished: A flag indicating if the file has already been written.
+    """
 
     def __init__(self, web_view, dest):
         self.web_view = web_view
@@ -219,6 +235,7 @@ class _Downloader(object):
         self.writer = MHTMLWriter()
         self.loaded_urls = {web_view.url()}
         self.pending_downloads = set()
+        self._finished = False
 
     def run(self):
         """Download and save the page.
@@ -251,6 +268,11 @@ class _Downloader(object):
             absolute_url = web_url.resolved(QUrl(element_url))
             self.fetch_url(absolute_url)
 
+        # Shortcut if no assets need to be downloaded, otherwise the file would
+        # never be saved
+        if not elements and not self.pending_downloads:
+            self.finish_file()
+
     def fetch_url(self, url):
         """Download the given url and add the file to the collection.
 
@@ -316,6 +338,9 @@ class _Downloader(object):
 
     def finish_file(self):
         """Save the file to the filename given in __init__."""
+        if self._finished:
+            return
+        self._finished = True
         log.misc.debug("All assets downloaded, ready to finish off!")
         with open(self.dest, "wb") as file_output:
             self.writer.write_to(file_output)

From 8eafa1a105ea6f6e8658bb7c337c703c4b25d833 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 23 Sep 2015 16:18:41 +0200
Subject: [PATCH 13/66] Also scan CSS in <style> tags and inline CSS

As both may contain external links too (@import, url(...))
---
 qutebrowser/misc/mhtml.py | 77 +++++++++++++++++++++++++++++++++++----
 1 file changed, 69 insertions(+), 8 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index e2bb0a4a5..4eece95c4 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -52,8 +52,10 @@ def _get_css_imports(data):
     The returned URLs are relative to the stylesheet's URL.
 
     Args:
-        data: The content of the stylesheet to scan as bytes.
+        data: The content of the stylesheet to scan.
     """
+    if isinstance(data, str):
+        data = data.encode("utf-8")
     urls = []
     for pattern in _CSS_URL_PATTERNS:
         for match in pattern.finditer(data):
@@ -225,7 +227,7 @@ class _Downloader(object):
         dest: Destination filename.
         writer: The MHTMLWriter object which is used to save the page.
         loaded_urls: A set of QUrls of finished asset downloads.
-        pending_downloads: A set of unfinished DownloadItems.
+        pending_downloads: A set of unfinished (url, DownloadItem) tuples.
         _finished: A flag indicating if the file has already been written.
     """
 
@@ -268,9 +270,28 @@ class _Downloader(object):
             absolute_url = web_url.resolved(QUrl(element_url))
             self.fetch_url(absolute_url)
 
+        styles = web_frame.findAllElements("style")
+        for style in styles:
+            if style.attribute("type", "text/css") != "text/css":
+                continue
+            for element_url in _get_css_imports(style.toPlainText()):
+                element_url = element_url.decode("ascii")
+                self.fetch_url(web_url.resolved(QUrl(element_url)))
+
+        # Search for references in inline styles
+        for element in web_frame.findAllElements("*"):
+            style = element.attribute("style")
+            if not style:
+                continue
+            for element_url in _get_css_imports(style):
+                element_url = element_url.decode("ascii")
+                self.fetch_url(web_url.resolved(QUrl(element_url)))
+
         # Shortcut if no assets need to be downloaded, otherwise the file would
-        # never be saved
-        if not elements and not self.pending_downloads:
+        # never be saved. Also might happen if the downloads are fast enough to
+        # complete before connecting their finished signal.
+        self.collect_zombies()
+        if not self.pending_downloads and not self._finished:
             self.finish_file()
 
     def fetch_url(self, url):
@@ -279,6 +300,8 @@ class _Downloader(object):
         Args:
             url: The file to download as QUrl.
         """
+        if url.scheme() == "data":
+            return
         # Prevent loading an asset twice
         if url in self.loaded_urls:
             return
@@ -288,9 +311,9 @@ class _Downloader(object):
 
         download_manager = objreg.get("download-manager", scope="window",
                                       window="current")
-        item = download_manager.get(url, fileobj=io.BytesIO(),
+        item = download_manager.get(url, fileobj=_NoCloseBytesIO(),
                                     auto_remove=True)
-        self.pending_downloads.add(item)
+        self.pending_downloads.add((url, item))
         item.finished.connect(
             functools.partial(self.finished, url, item))
         item.error.connect(
@@ -305,7 +328,7 @@ class _Downloader(object):
             url: The original url of the asset as QUrl.
             item: The DownloadItem given by the DownloadManager
         """
-        self.pending_downloads.remove(item)
+        self.pending_downloads.remove((url, item))
         mime = item.raw_headers.get(b"Content-Type", b"")
         mime = mime.decode("ascii", "ignore")
 
@@ -319,6 +342,7 @@ class _Downloader(object):
         encode = E_QUOPRI if mime.startswith("text/") else E_BASE64
         self.writer.add_file(url.toString(), item.fileobj.getvalue(), mime,
                              encode)
+        item.fileobj.actual_close()
         if self.pending_downloads:
             return
         self.finish_file()
@@ -330,7 +354,14 @@ class _Downloader(object):
             url: The orignal url of the asset as QUrl.
             item: The DownloadItem given by the DownloadManager.
         """
-        self.pending_downloads.remove(item)
+        try:
+            self.pending_downloads.remove((url, item))
+        except KeyError:
+            # This might happen if .collect_zombies() calls .finished() and the
+            # error handler will be called after .collect_zombies
+            log.misc.debug("Oops! Download already gone: %s", item)
+            return
+        item.fileobj.actual_close()
         self.writer.add_file(url.toString(), b"")
         if self.pending_downloads:
             return
@@ -339,6 +370,7 @@ class _Downloader(object):
     def finish_file(self):
         """Save the file to the filename given in __init__."""
         if self._finished:
+            log.misc.debug("finish_file called twice, ignored!")
             return
         self._finished = True
         log.misc.debug("All assets downloaded, ready to finish off!")
@@ -346,6 +378,35 @@ class _Downloader(object):
             self.writer.write_to(file_output)
         message.info("current", "Page saved as {}".format(self.dest), True)
 
+    def collect_zombies(self):
+        """Collect done downloads and add their data to the MHTML file.
+
+        This is needed if a download finishes before attaching its
+        finished signal.
+        """
+        items = set((url, item) for url, item in self.pending_downloads
+                    if item.done)
+        log.misc.debug("Zombie downloads: %s", items)
+        for url, item in items:
+            self.finished(url, item)
+
+
+class _NoCloseBytesIO(io.BytesIO):
+
+    """BytesIO that can't be .closed()
+
+    This is needed to prevent the downloadmanager from closing the stream, thus
+    discarding the data.
+    """
+
+    def close(self):
+        """Do nothing."""
+        pass
+
+    def actual_close(self):
+        """Close the stream."""
+        super().close()
+
 
 def start_download(dest):
     """Start downloading the current page and all assets to a MHTML file.

From 05cc4b9650d767d6d2be0e01d0f4b1e8fda07866 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 23 Sep 2015 16:25:21 +0200
Subject: [PATCH 14/66] Change boundary

This version contains a sequence that is illegal in quoted-printable
and thus safe from accidentally appearing in a website.
---
 qutebrowser/misc/mhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 4eece95c4..01f8f5cc9 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -130,7 +130,7 @@ class MHTMLWriter(object):
         _files: Mapping of location->_File struct.
     """
 
-    BOUNDARY = b"---qute-mhtml-" + str(uuid4()).encode("ascii")
+    BOUNDARY = b"---=_qute-" + str(uuid4()).encode("ascii")
 
     def __init__(self, root_content=None, content_location=None,
                  content_type=None):

From 64c74bde904a1e1ba39aaf4108d62e1a3a06ee5d Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 23 Sep 2015 17:18:26 +0200
Subject: [PATCH 15/66] Fix pylint for _NoCloseBytesIO

---
 qutebrowser/misc/mhtml.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 01f8f5cc9..6661fb1ad 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -391,11 +391,11 @@ class _Downloader(object):
             self.finished(url, item)
 
 
-class _NoCloseBytesIO(io.BytesIO):
+class _NoCloseBytesIO(io.BytesIO):  # pylint: disable=no-init
 
-    """BytesIO that can't be .closed()
+    """BytesIO that can't be .closed().
 
-    This is needed to prevent the downloadmanager from closing the stream, thus
+    This is needed to prevent the DownloadManager from closing the stream, thus
     discarding the data.
     """
 

From f58f6f24ee5c996c66c3077e53397a997fef0f8c Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 23 Sep 2015 18:10:14 +0200
Subject: [PATCH 16/66] Use email.mime instead of manually writing the msg

---
 qutebrowser/misc/mhtml.py | 69 +++++++++++++++++----------------------
 1 file changed, 30 insertions(+), 39 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 6661fb1ad..e35a012e8 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -27,6 +27,9 @@ import re
 from collections import namedtuple
 from base64 import b64encode
 from uuid import uuid4
+from email import policy
+from email.mime.multipart import MIMEMultipart
+from email.generator import BytesGenerator
 
 from PyQt5.QtCore import QUrl
 
@@ -65,6 +68,9 @@ def _get_css_imports(data):
     return urls
 
 
+MHTMLPolicy = policy.default.clone(linesep="\r\n", max_line_length=0)
+
+
 def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
     """Just like b64encode, except that it breaks long lines.
 
@@ -130,7 +136,7 @@ class MHTMLWriter(object):
         _files: Mapping of location->_File struct.
     """
 
-    BOUNDARY = b"---=_qute-" + str(uuid4()).encode("ascii")
+    BOUNDARY = "---=_qute-" + str(uuid4())
 
     def __init__(self, root_content=None, content_location=None,
                  content_type=None):
@@ -169,53 +175,38 @@ class MHTMLWriter(object):
         Args:
             fp: The file-object, openend in "wb" mode.
         """
-        self._output_header(fp)
-        self._output_root_file(fp)
+        msg = MIMEMultipart("related", self.BOUNDARY)
+
+        root = self._create_root_file()
+        msg.attach(root)
+
         for file_data in self._files.values():
-            self._output_file(fp, file_data)
-        fp.write(b"\r\n--")
-        fp.write(self.BOUNDARY)
-        fp.write(b"--")
+            msg.attach(self._create_file(file_data))
 
-    def _output_header(self, fp):
-        """Output only the header to the given fileobject."""
-        if self.content_location is None:
-            raise ValueError("content_location must be set")
-        if self.content_type is None:
-            raise ValueError("content_type must be set for the root document")
+        gen = BytesGenerator(fp, policy=MHTMLPolicy)
+        gen.flatten(msg)
 
-        fp.write(b"Content-Location: ")
-        fp.write(self.content_location.encode("utf-8"))
-        fp.write(b'\r\nContent-Type: multipart/related;boundary="')
-        fp.write(self.BOUNDARY)
-        fp.write(b'";type="')
-        fp.write(self.content_type.encode("utf-8"))
-        fp.write(b'"\r\n\r\n')
-
-    def _output_root_file(self, fp):
-        """Output the root document to the fileobject."""
+    def _create_root_file(self):
+        """Return the root document as MIMEMultipart."""
         root_file = _File(
             content=self.root_content, content_type=self.content_type,
             content_location=self.content_location, transfer_encoding=E_QUOPRI,
         )
-        self._output_file(fp, root_file)
+        return self._create_file(root_file)
 
-    def _output_file(self, fp, file_struct):
-        """Output the single given file to the fileobject."""
-        fp.write(b"--")
-        fp.write(self.BOUNDARY)
-        fp.write(b"\r\nContent-Location: ")
-        fp.write(file_struct.content_location.encode("utf-8"))
-        if file_struct.content_type is not None:
-            fp.write(b"\r\nContent-Type: ")
-            fp.write(file_struct.content_type.encode("utf-8"))
-        encoding_name, encoding_func = file_struct.transfer_encoding
+    def _create_file(self, f):
+        """Return the single given file as MIMEMultipart."""
+        msg = MIMEMultipart()
+        msg["Content-Location"] = f.content_location
+        # Get rid of the default type multipart/mixed
+        del msg["Content-Type"]
+        if f.content_type:
+            msg.set_type(f.content_type)
+        encoding_name, encoding_func = f.transfer_encoding
         if encoding_name:
-            fp.write(b"\r\nContent-Transfer-Encoding: ")
-            fp.write(encoding_name.encode("utf-8"))
-        fp.write(b"\r\n\r\n")
-        fp.write(encoding_func(file_struct.content))
-        fp.write(b"\r\n\r\n")
+            msg["Content-Transfer-Encoding"] = encoding_name
+        msg.set_payload(encoding_func(f.content).decode("ascii"))
+        return msg
 
 
 class _Downloader(object):

From 8593144fa78ca950d6acae23732fed70184f4769 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 12:38:52 +0200
Subject: [PATCH 17/66] Make _path_suggestion public

---
 qutebrowser/browser/commands.py  | 6 ++----
 qutebrowser/browser/downloads.py | 6 +++---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 6477bf3f4..f1b3f4661 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -37,14 +37,12 @@ import pygments.formatters
 
 from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
 from qutebrowser.config import config, configexc
-from qutebrowser.browser import webelem, inspector, urlmarks
+from qutebrowser.browser import webelem, inspector, urlmarks, downloads
 from qutebrowser.keyinput import modeman
 from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
                                objreg, utils)
 from qutebrowser.utils.usertypes import KeyMode
 from qutebrowser.misc import editor, guiprocess, mhtml
-from qutebrowser.browser.downloads import (_path_suggestion as
-                                           download_path_suggestion)
 
 
 class CommandDispatcher:
@@ -1172,7 +1170,7 @@ class CommandDispatcher:
             q.text = "Save page to: "
             q.mode = usertypes.PromptMode.text
             q.completed.connect(q.deleteLater)
-            q.default = download_path_suggestion(suggested_fn)
+            q.default = downloads.path_suggestion(suggested_fn)
             q.answered.connect(mhtml.start_download)
             message_bridge = objreg.get("message-bridge", scope="window",
                                         window=self._win_id)
diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index c446b6c55..a620970ed 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -70,7 +70,7 @@ def _download_dir():
         return directory
 
 
-def _path_suggestion(filename):
+def path_suggestion(filename):
     """Get the suggested file path.
 
     Args:
@@ -750,7 +750,7 @@ class DownloadManager(QAbstractListModel):
             suggested_fn = utils.force_encoding(suggested_fn, encoding)
 
         q = self._prepare_question()
-        q.default = _path_suggestion(suggested_fn)
+        q.default = path_suggestion(suggested_fn)
         message_bridge = objreg.get('message-bridge', scope='window',
                                     window=self._win_id)
         q.answered.connect(
@@ -839,7 +839,7 @@ class DownloadManager(QAbstractListModel):
             download.autoclose = False
         else:
             q = self._prepare_question()
-            q.default = _path_suggestion(suggested_filename)
+            q.default = path_suggestion(suggested_filename)
             q.answered.connect(download.set_filename)
             q.cancelled.connect(download.cancel)
             download.cancelled.connect(q.abort)

From 83aee4fad5a2c116394a8c5ac1b2943187e9038e Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 12:40:40 +0200
Subject: [PATCH 18/66] Rename on_meta_data_change to on_meta_data_changed

---
 qutebrowser/browser/downloads.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index a620970ed..3478b0a87 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -356,7 +356,7 @@ class DownloadItem(QObject):
         reply.finished.connect(self.on_reply_finished)
         reply.error.connect(self.on_reply_error)
         reply.readyRead.connect(self.on_ready_read)
-        reply.metaDataChanged.connect(self.on_meta_data_change)
+        reply.metaDataChanged.connect(self.on_meta_data_changed)
         self.retry_info = RetryInfo(request=reply.request(),
                                     manager=reply.manager())
         if not self.fileobj:
@@ -600,7 +600,7 @@ class DownloadItem(QObject):
             self._buffer.write(data)
 
     @pyqtSlot()
-    def on_meta_data_change(self):
+    def on_meta_data_changed(self):
         """Update the download's metadata."""
         if self.reply is None:
             return

From a3cc71e317c985464e45c7f9fe2b369b1ac32061 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 12:52:10 +0200
Subject: [PATCH 19/66] Don't from-import functions/classes

---
 qutebrowser/misc/mhtml.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index e35a012e8..4f7aee788 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -24,20 +24,20 @@ import io
 import os
 import re
 
-from collections import namedtuple
-from base64 import b64encode
-from uuid import uuid4
-from email import policy
-from email.mime.multipart import MIMEMultipart
-from email.generator import BytesGenerator
+import collections
+import base64
+import uuid
+from email import policy, generator
+from email.mime import multipart
 
 from PyQt5.QtCore import QUrl
 
 from qutebrowser.utils import log, objreg, message
 
 
-_File = namedtuple("_File",
-                   "content content_type content_location transfer_encoding")
+_File = collections.namedtuple("_File",
+                               ["content", "content_type", "content_location",
+                                "transfer_encoding"])
 
 
 _CSS_URL_PATTERNS = [re.compile(x) for x in [
@@ -78,7 +78,7 @@ def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
         maxlen: Maximum length of a line, not including the line separator.
         linesep: Line separator to use as bytes.
     """
-    encoded = b64encode(data)
+    encoded = base64.b64encode(data)
     result = []
     for i in range(0, len(encoded), maxlen):
         result.append(encoded[i:i + maxlen])
@@ -136,7 +136,7 @@ class MHTMLWriter(object):
         _files: Mapping of location->_File struct.
     """
 
-    BOUNDARY = "---=_qute-" + str(uuid4())
+    BOUNDARY = "---=_qute-" + str(uuid.uuid4())
 
     def __init__(self, root_content=None, content_location=None,
                  content_type=None):
@@ -175,7 +175,7 @@ class MHTMLWriter(object):
         Args:
             fp: The file-object, openend in "wb" mode.
         """
-        msg = MIMEMultipart("related", self.BOUNDARY)
+        msg = multipart.MIMEMultipart("related", self.BOUNDARY)
 
         root = self._create_root_file()
         msg.attach(root)
@@ -183,7 +183,7 @@ class MHTMLWriter(object):
         for file_data in self._files.values():
             msg.attach(self._create_file(file_data))
 
-        gen = BytesGenerator(fp, policy=MHTMLPolicy)
+        gen = generator.BytesGenerator(fp, policy=MHTMLPolicy)
         gen.flatten(msg)
 
     def _create_root_file(self):
@@ -196,7 +196,7 @@ class MHTMLWriter(object):
 
     def _create_file(self, f):
         """Return the single given file as MIMEMultipart."""
-        msg = MIMEMultipart()
+        msg = multipart.MIMEMultipart()
         msg["Content-Location"] = f.content_location
         # Get rid of the default type multipart/mixed
         del msg["Content-Type"]

From e5bfb9884b701b1a69459a1ba73dcc75348765a9 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 13:22:53 +0200
Subject: [PATCH 20/66] Use WebElementWrapper instead of QWebElement

* also don't derive from object
* also set the _used flag on _Downloader
---
 qutebrowser/misc/mhtml.py | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 4f7aee788..c69b73fa4 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -32,6 +32,7 @@ from email.mime import multipart
 
 from PyQt5.QtCore import QUrl
 
+from qutebrowser.browser import webelem
 from qutebrowser.utils import log, objreg, message
 
 
@@ -125,7 +126,7 @@ E_QUOPRI = ("quoted-printable", _rn_quopri)
 """Encode the file using MIME quoted-printable encoding."""
 
 
-class MHTMLWriter(object):
+class MHTMLWriter():
 
     """A class for outputting multiple files to a MHTML document.
 
@@ -209,7 +210,7 @@ class MHTMLWriter(object):
         return msg
 
 
-class _Downloader(object):
+class _Downloader():
 
     """A class to download whole websites.
 
@@ -220,6 +221,7 @@ class _Downloader(object):
         loaded_urls: A set of QUrls of finished asset downloads.
         pending_downloads: A set of unfinished (url, DownloadItem) tuples.
         _finished: A flag indicating if the file has already been written.
+        _used: A flag indicating if the downloader has already been used.
     """
 
     def __init__(self, web_view, dest):
@@ -229,6 +231,7 @@ class _Downloader(object):
         self.loaded_urls = {web_view.url()}
         self.pending_downloads = set()
         self._finished = False
+        self._used = False
 
     def run(self):
         """Download and save the page.
@@ -236,6 +239,9 @@ class _Downloader(object):
         The object must not be reused, you should create a new one if
         you want to download another page.
         """
+        if self._used:
+            raise ValueError("Downloader already used")
+        self._used = True
         web_url = self.web_view.url()
         web_frame = self.web_view.page().mainFrame()
 
@@ -252,10 +258,12 @@ class _Downloader(object):
                     web_frame.findAllElements("img"))
 
         for element in elements:
-            element_url = element.attribute("src")
-            if not element_url:
-                element_url = element.attribute("href")
-            if not element_url:
+            element = webelem.WebElementWrapper(element)
+            if "src" in element:
+                element_url = element["src"]
+            elif "href" in element:
+                element_url = element["href"]
+            else:
                 # Might be a local <script> tag or something else
                 continue
             absolute_url = web_url.resolved(QUrl(element_url))
@@ -263,17 +271,19 @@ class _Downloader(object):
 
         styles = web_frame.findAllElements("style")
         for style in styles:
-            if style.attribute("type", "text/css") != "text/css":
+            style = webelem.WebElementWrapper(style)
+            if "type" in style and style["type"] != "text/css":
                 continue
-            for element_url in _get_css_imports(style.toPlainText()):
+            for element_url in _get_css_imports(str(style)):
                 element_url = element_url.decode("ascii")
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
         # Search for references in inline styles
         for element in web_frame.findAllElements("*"):
-            style = element.attribute("style")
-            if not style:
+            element = webelem.WebElementWrapper(element)
+            if "style" not in element:
                 continue
+            style = element["style"]
             for element_url in _get_css_imports(style):
                 element_url = element_url.decode("ascii")
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
@@ -367,7 +377,7 @@ class _Downloader(object):
         log.misc.debug("All assets downloaded, ready to finish off!")
         with open(self.dest, "wb") as file_output:
             self.writer.write_to(file_output)
-        message.info("current", "Page saved as {}".format(self.dest), True)
+        message.info("current", "Page saved as {}".format(self.dest))
 
     def collect_zombies(self):
         """Collect done downloads and add their data to the MHTML file.

From d3a21927f27bc4c097afc6f5d154e685ebca3a2b Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 13:30:47 +0200
Subject: [PATCH 21/66] Remove default values in MHTMLWriter.__init__

---
 qutebrowser/misc/mhtml.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index c69b73fa4..25972d127 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -139,8 +139,7 @@ class MHTMLWriter():
 
     BOUNDARY = "---=_qute-" + str(uuid.uuid4())
 
-    def __init__(self, root_content=None, content_location=None,
-                 content_type=None):
+    def __init__(self, root_content, content_location, content_type):
         self.root_content = root_content
         self.content_location = content_location
         self.content_type = content_type
@@ -227,7 +226,7 @@ class _Downloader():
     def __init__(self, web_view, dest):
         self.web_view = web_view
         self.dest = dest
-        self.writer = MHTMLWriter()
+        self.writer = None
         self.loaded_urls = {web_view.url()}
         self.pending_downloads = set()
         self._finished = False
@@ -245,12 +244,14 @@ class _Downloader():
         web_url = self.web_view.url()
         web_frame = self.web_view.page().mainFrame()
 
-        self.writer.root_content = web_frame.toHtml().encode("utf-8")
-        self.writer.content_location = web_url.toString()
-        # I've found no way of getting the content type of a QWebView, but
-        # since we're using .toHtml, it's probably safe to say that the
-        # content-type is HTML
-        self.writer.content_type = 'text/html; charset="UTF-8"'
+        self.writer = MHTMLWriter(
+            web_frame.toHtml().encode("utf-8"),
+            content_location = web_url.toString(),
+            # I've found no way of getting the content type of a QWebView, but
+            # since we're using .toHtml, it's probably safe to say that the
+            # content-type is HTML
+            content_type = 'text/html; charset="UTF-8"',
+        )
         # Currently only downloading <link> (stylesheets), <script>
         # (javascript) and <img> (image) elements.
         elements = (web_frame.findAllElements("link") +

From ba81332d45c934d4b988c4e1408c7a5b4e1e4d46 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 13:48:58 +0200
Subject: [PATCH 22/66] _get_css_imports now works on strings only

This also means that it returns strings, making the calls to .decode
unneeded.
---
 qutebrowser/misc/mhtml.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 25972d127..4c77a9829 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -42,11 +42,11 @@ _File = collections.namedtuple("_File",
 
 
 _CSS_URL_PATTERNS = [re.compile(x) for x in [
-    rb"@import '(?P<url>[^']+)'",
-    rb'@import "(?P<url>[^"]+)"',
-    rb'''url\((?P<url>[^'"][^)]*)\)''',
-    rb'url\("(?P<url>[^"]+)"\)',
-    rb"url\('(?P<url>[^']+)'\)",
+    r"@import '(?P<url>[^']+)'",
+    r'@import "(?P<url>[^"]+)"',
+    r'''url\((?P<url>[^'"][^)]*)\)''',
+    r'url\("(?P<url>[^"]+)"\)',
+    r"url\('(?P<url>[^']+)'\)",
 ]]
 
 
@@ -56,10 +56,8 @@ def _get_css_imports(data):
     The returned URLs are relative to the stylesheet's URL.
 
     Args:
-        data: The content of the stylesheet to scan.
+        data: The content of the stylesheet to scan as string.
     """
-    if isinstance(data, str):
-        data = data.encode("utf-8")
     urls = []
     for pattern in _CSS_URL_PATTERNS:
         for match in pattern.finditer(data):
@@ -246,11 +244,11 @@ class _Downloader():
 
         self.writer = MHTMLWriter(
             web_frame.toHtml().encode("utf-8"),
-            content_location = web_url.toString(),
+            content_location=web_url.toString(),
             # I've found no way of getting the content type of a QWebView, but
             # since we're using .toHtml, it's probably safe to say that the
             # content-type is HTML
-            content_type = 'text/html; charset="UTF-8"',
+            content_type='text/html; charset="UTF-8"',
         )
         # Currently only downloading <link> (stylesheets), <script>
         # (javascript) and <img> (image) elements.
@@ -276,7 +274,6 @@ class _Downloader():
             if "type" in style and style["type"] != "text/css":
                 continue
             for element_url in _get_css_imports(str(style)):
-                element_url = element_url.decode("ascii")
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
         # Search for references in inline styles
@@ -286,7 +283,6 @@ class _Downloader():
                 continue
             style = element["style"]
             for element_url in _get_css_imports(style):
-                element_url = element_url.decode("ascii")
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
         # Shortcut if no assets need to be downloaded, otherwise the file would
@@ -335,9 +331,19 @@ class _Downloader():
         mime = mime.decode("ascii", "ignore")
 
         if mime.lower() == "text/css":
-            import_urls = _get_css_imports(item.fileobj.getvalue())
+            # We can't always assume that CSS files are UTF-8, but CSS files
+            # shouldn't contain many non-ASCII characters anyway (in most
+            # cases). Using "ignore" lets us decode the file even if it's
+            # invalid UTF-8 data.
+            # The file written to the MHTML file won't be modified by this
+            # decoding, since there we're taking the original bytestream.
+            try:
+                css_string = item.fileobj.getvalue().decode("utf-8")
+            except UnicodeDecodeError:
+                log.misc.warning("Invalid UTF-8 data in %s", url)
+                css_string = item.fileobj.getvalue().decode("utf-8", "ignore")
+            import_urls = _get_css_imports(css_string)
             for import_url in import_urls:
-                import_url = import_url.decode("ascii")
                 absolute_url = url.resolved(QUrl(import_url))
                 self.fetch_url(absolute_url)
 

From a63aed5965a0987d29517d4b715a32bfffc92c1f Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 14:07:12 +0200
Subject: [PATCH 23/66] Use email.encoders instead of own encoder function

---
 qutebrowser/misc/mhtml.py | 60 ++++-----------------------------------
 1 file changed, 5 insertions(+), 55 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 4c77a9829..c4d6d6ebc 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -25,9 +25,8 @@ import os
 import re
 
 import collections
-import base64
 import uuid
-from email import policy, generator
+from email import policy, generator, encoders
 from email.mime import multipart
 
 from PyQt5.QtCore import QUrl
@@ -70,57 +69,10 @@ def _get_css_imports(data):
 MHTMLPolicy = policy.default.clone(linesep="\r\n", max_line_length=0)
 
 
-def _chunked_base64(data, maxlen=76, linesep=b"\r\n"):
-    """Just like b64encode, except that it breaks long lines.
-
-    Args:
-        maxlen: Maximum length of a line, not including the line separator.
-        linesep: Line separator to use as bytes.
-    """
-    encoded = base64.b64encode(data)
-    result = []
-    for i in range(0, len(encoded), maxlen):
-        result.append(encoded[i:i + maxlen])
-    return linesep.join(result)
-
-
-def _rn_quopri(data):
-    """Return a quoted-printable representation of data."""
-    # See RFC 2045 https://tools.ietf.org/html/rfc2045#section-6.7
-    # The stdlib version in the quopri module has inconsistencies with line
-    # endings and breaks up character escapes over multiple lines, which isn't
-    # understood by qute and leads to jumbled text
-    maxlen = 76
-    whitespace = {ord(b"\t"), ord(b" ")}
-    output = []
-    current_line = b""
-    for byte in data:
-        # Literal representation according to (2) and (3)
-        if (ord(b"!") <= byte <= ord(b"<") or ord(b">") <= byte <= ord(b"~")
-                or byte in whitespace):
-            current_line += bytes([byte])
-        else:
-            current_line += b"=" + "{:02X}".format(byte).encode("ascii")
-        if len(current_line) >= maxlen:
-            # We need to account for the = character
-            split = [current_line[:maxlen - 1], current_line[maxlen - 1:]]
-            quoted_pos = split[0].rfind(b"=")
-            if quoted_pos + 2 >= maxlen - 1:
-                split[0], token = split[0][:quoted_pos], split[0][quoted_pos:]
-                split[1] = token + split[1]
-            current_line = split[1]
-            output.append(split[0] + b"=")
-    output.append(current_line)
-    return b"\r\n".join(output)
-
-
-E_NONE = (None, lambda x: x)
-"""No transfer encoding, copy the bytes from input to output"""
-
-E_BASE64 = ("base64", _chunked_base64)
+E_BASE64 = encoders.encode_base64
 """Encode the file using base64 encoding"""
 
-E_QUOPRI = ("quoted-printable", _rn_quopri)
+E_QUOPRI = encoders.encode_quopri
 """Encode the file using MIME quoted-printable encoding."""
 
 
@@ -200,10 +152,8 @@ class MHTMLWriter():
         del msg["Content-Type"]
         if f.content_type:
             msg.set_type(f.content_type)
-        encoding_name, encoding_func = f.transfer_encoding
-        if encoding_name:
-            msg["Content-Transfer-Encoding"] = encoding_name
-        msg.set_payload(encoding_func(f.content).decode("ascii"))
+        msg.set_payload(f.content)
+        f.transfer_encoding(msg)
         return msg
 
 

From cb477a26235fe6f60a434f7bf6e29d27f4579703 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 14:27:17 +0200
Subject: [PATCH 24/66] Decode headers with ISO-8859-1 instead of ASCII

---
 qutebrowser/misc/mhtml.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index c4d6d6ebc..cc7e7186e 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -278,7 +278,17 @@ class _Downloader():
         """
         self.pending_downloads.remove((url, item))
         mime = item.raw_headers.get(b"Content-Type", b"")
-        mime = mime.decode("ascii", "ignore")
+
+        # Note that this decoding always works and doesn't produce errors
+        # RFC 7230 (https://tools.ietf.org/html/rfc7230) states:
+        # Historically, HTTP has allowed field content with text in the
+        # ISO-8859-1 charset [ISO-8859-1], supporting other charsets only
+        # through use of [RFC2047] encoding.  In practice, most HTTP header
+        # field values use only a subset of the US-ASCII charset [USASCII].
+        # Newly defined header fields SHOULD limit their field values to
+        # US-ASCII octets.  A recipient SHOULD treat other octets in field
+        # content (obs-text) as opaque data.
+        mime = mime.decode("iso-8859-1")
 
         if mime.lower() == "text/css":
             # We can't always assume that CSS files are UTF-8, but CSS files

From afa2f339e66c0bdd55bb5d8b01005031a9acb609 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 14:37:50 +0200
Subject: [PATCH 25/66] mhtm: use downloads logger instead of misc

---
 qutebrowser/misc/mhtml.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index cc7e7186e..228b31648 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -255,7 +255,7 @@ class _Downloader():
             return
         self.loaded_urls.add(url)
 
-        log.misc.debug("loading asset at %s", url)
+        log.downloads.debug("loading asset at %s", url)
 
         download_manager = objreg.get("download-manager", scope="window",
                                       window="current")
@@ -300,7 +300,7 @@ class _Downloader():
             try:
                 css_string = item.fileobj.getvalue().decode("utf-8")
             except UnicodeDecodeError:
-                log.misc.warning("Invalid UTF-8 data in %s", url)
+                log.downloads.warning("Invalid UTF-8 data in %s", url)
                 css_string = item.fileobj.getvalue().decode("utf-8", "ignore")
             import_urls = _get_css_imports(css_string)
             for import_url in import_urls:
@@ -327,7 +327,7 @@ class _Downloader():
         except KeyError:
             # This might happen if .collect_zombies() calls .finished() and the
             # error handler will be called after .collect_zombies
-            log.misc.debug("Oops! Download already gone: %s", item)
+            log.downloads.debug("Oops! Download already gone: %s", item)
             return
         item.fileobj.actual_close()
         self.writer.add_file(url.toString(), b"")
@@ -338,10 +338,10 @@ class _Downloader():
     def finish_file(self):
         """Save the file to the filename given in __init__."""
         if self._finished:
-            log.misc.debug("finish_file called twice, ignored!")
+            log.downloads.debug("finish_file called twice, ignored!")
             return
         self._finished = True
-        log.misc.debug("All assets downloaded, ready to finish off!")
+        log.downloads.debug("All assets downloaded, ready to finish off!")
         with open(self.dest, "wb") as file_output:
             self.writer.write_to(file_output)
         message.info("current", "Page saved as {}".format(self.dest))
@@ -354,7 +354,7 @@ class _Downloader():
         """
         items = set((url, item) for url, item in self.pending_downloads
                     if item.done)
-        log.misc.debug("Zombie downloads: %s", items)
+        log.downloads.debug("Zombie downloads: %s", items)
         for url, item in items:
             self.finished(url, item)
 

From 5fcbc839bbfcaecadffdcf71836d8685d11793b0 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 17:55:18 +0200
Subject: [PATCH 26/66] Allow many spaces and tabs after @import in CSS

---
 qutebrowser/misc/mhtml.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 228b31648..681a563c8 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -41,8 +41,8 @@ _File = collections.namedtuple("_File",
 
 
 _CSS_URL_PATTERNS = [re.compile(x) for x in [
-    r"@import '(?P<url>[^']+)'",
-    r'@import "(?P<url>[^"]+)"',
+    r"@import\s+'(?P<url>[^']+)'",
+    r'@import\s+"(?P<url>[^"]+)"',
     r'''url\((?P<url>[^'"][^)]*)\)''',
     r'url\("(?P<url>[^"]+)"\)',
     r"url\('(?P<url>[^']+)'\)",

From bf90c8c06b6ff21b8f19ddb739b033f37b70d529 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 17:56:33 +0200
Subject: [PATCH 27/66] Add tests for mhtml

This also makes the output of MHTMLWriter deterministic, by

1) Setting the boundary at object creation, allowing uuid.uuid4 to be
   monkey patched

2) Outputting the files in sorted order (sorted by location), as python
   dicts are unordered by default.
---
 qutebrowser/misc/mhtml.py     |   7 +-
 tests/unit/misc/test_mhtml.py | 268 ++++++++++++++++++++++++++++++++++
 2 files changed, 271 insertions(+), 4 deletions(-)
 create mode 100644 tests/unit/misc/test_mhtml.py

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 681a563c8..56cb974a7 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -87,8 +87,6 @@ class MHTMLWriter():
         _files: Mapping of location->_File struct.
     """
 
-    BOUNDARY = "---=_qute-" + str(uuid.uuid4())
-
     def __init__(self, root_content, content_location, content_type):
         self.root_content = root_content
         self.content_location = content_location
@@ -125,12 +123,13 @@ class MHTMLWriter():
         Args:
             fp: The file-object, openend in "wb" mode.
         """
-        msg = multipart.MIMEMultipart("related", self.BOUNDARY)
+        msg = multipart.MIMEMultipart("related",
+                                      "---=_qute-{}".format(uuid.uuid4()))
 
         root = self._create_root_file()
         msg.attach(root)
 
-        for file_data in self._files.values():
+        for _, file_data in sorted(self._files.items()):
             msg.attach(self._create_file(file_data))
 
         gen = generator.BytesGenerator(fp, policy=MHTMLPolicy)
diff --git a/tests/unit/misc/test_mhtml.py b/tests/unit/misc/test_mhtml.py
new file mode 100644
index 000000000..8f0a59379
--- /dev/null
+++ b/tests/unit/misc/test_mhtml.py
@@ -0,0 +1,268 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+import io
+import textwrap
+import re
+
+import pytest
+
+from qutebrowser.misc import mhtml
+
+@pytest.fixture(autouse=True)
+def patch_uuid(monkeypatch):
+    monkeypatch.setattr("uuid.uuid4", lambda: "UUID")
+
+
+class Checker:
+
+    """A helper to check mhtml output.
+
+    Attrs:
+        fp: A BytesIO object for passing to MHTMLWriter.write_to.
+    """
+
+    def __init__(self):
+        self.fp = io.BytesIO()
+
+    @property
+    def value(self):
+        return self.fp.getvalue()
+
+    def expect(self, expected):
+        actual = self.value.decode("ascii")
+        # Make sure there are no stray \r or \n
+        assert re.search(r"\r[^\n]", actual) is None
+        assert re.search(r"[^\r]\n", actual) is None
+        actual = actual.replace("\r\n", "\n")
+        expected = textwrap.dedent(expected).lstrip("\n")
+        assert expected == actual
+
+
+@pytest.fixture
+def checker():
+    return Checker()
+
+
+def test_quoted_printable_umlauts(checker):
+    content = "Die süße Hündin läuft in die Höhle des Bären"
+    content = content.encode("iso-8859-1")
+    writer = mhtml.MHTMLWriter(root_content=content,
+                               content_location="localhost",
+                               content_type="text/plain")
+    writer.write_to(checker.fp)
+    checker.expect("""
+    Content-Type: multipart/related; boundary="---=_qute-UUID"
+    MIME-Version: 1.0
+
+    -----=_qute-UUID
+    Content-Location: localhost
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    Die=20s=FC=DFe=20H=FCndin=20l=E4uft=20in=20die=20H=F6hle=20des=20B=E4ren
+    -----=_qute-UUID--
+    """)
+
+
+@pytest.mark.parametrize("header, value", [
+    ("content_location", "http://brötli.com"),
+    ("content_type", "text/pläin"),
+])
+def test_refuses_non_ascii_header_value(checker, header, value):
+    defaults = {
+        "root_content": b"",
+        "content_location": "http://example.com",
+        "content_type": "text/plain",
+    }
+    defaults[header] = value
+    writer = mhtml.MHTMLWriter(**defaults)
+    with pytest.raises(UnicodeEncodeError) as e:
+        writer.write_to(checker.fp)
+        assert "'ascii' codec can't encode" in str(e)
+
+
+def test_file_encoded_as_base64(checker):
+    content = b"Image file attached"
+    writer = mhtml.MHTMLWriter(root_content=content, content_type="text/plain",
+                               content_location="http://example.com")
+    writer.add_file(location="http://a.example.com/image.png",
+                    content="\U0001F601 image data".encode("utf-8"),
+                    content_type="image/png",
+                    transfer_encoding=mhtml.E_BASE64)
+    writer.write_to(checker.fp)
+    checker.expect("""
+    Content-Type: multipart/related; boundary="---=_qute-UUID"
+    MIME-Version: 1.0
+
+    -----=_qute-UUID
+    Content-Location: http://example.com
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    Image=20file=20attached
+    -----=_qute-UUID
+    Content-Location: http://a.example.com/image.png
+    MIME-Version: 1.0
+    Content-Type: image/png
+    Content-Transfer-Encoding: base64
+
+    8J+YgSBpbWFnZSBkYXRh
+
+    -----=_qute-UUID--
+    """)
+
+
+@pytest.mark.parametrize("transfer_encoding", [mhtml.E_BASE64, mhtml.E_QUOPRI],
+                         ids=["base64", "quoted-printable"])
+def test_payload_lines_wrap(checker, transfer_encoding):
+    payload = b"1234567890" * 10
+    writer = mhtml.MHTMLWriter(root_content=b"", content_type="text/plain",
+                               content_location="http://example.com")
+    writer.add_file(location="http://example.com/payload", content=payload,
+                    content_type="text/plain",
+                    transfer_encoding=transfer_encoding)
+    writer.write_to(checker.fp)
+    for line in checker.value.split(b"\r\n"):
+        assert len(line) < 77
+
+
+def test_files_appear_sorted(checker):
+    writer = mhtml.MHTMLWriter(root_content=b"root file",
+                               content_type="text/plain",
+                               content_location="http://www.example.com/")
+    for subdomain in "ahgbizt":
+        writer.add_file(location="http://{}.example.com/".format(subdomain),
+                        content="file {}".format(subdomain).encode("utf-8"),
+                        content_type="text/plain",
+                        transfer_encoding=mhtml.E_QUOPRI)
+    writer.write_to(checker.fp)
+    checker.expect("""
+    Content-Type: multipart/related; boundary="---=_qute-UUID"
+    MIME-Version: 1.0
+
+    -----=_qute-UUID
+    Content-Location: http://www.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    root=20file
+    -----=_qute-UUID
+    Content-Location: http://a.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20a
+    -----=_qute-UUID
+    Content-Location: http://b.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20b
+    -----=_qute-UUID
+    Content-Location: http://g.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20g
+    -----=_qute-UUID
+    Content-Location: http://h.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20h
+    -----=_qute-UUID
+    Content-Location: http://i.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20i
+    -----=_qute-UUID
+    Content-Location: http://t.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20t
+    -----=_qute-UUID
+    Content-Location: http://z.example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20z
+    -----=_qute-UUID--
+    """)
+
+
+def test_empty_content_type(checker):
+    writer = mhtml.MHTMLWriter(root_content=b"",
+                               content_location="http://example.com/",
+                               content_type="text/plain")
+    writer.add_file("http://example.com/file", b"file content")
+    writer.write_to(checker.fp)
+    checker.expect("""
+    Content-Type: multipart/related; boundary="---=_qute-UUID"
+    MIME-Version: 1.0
+
+    -----=_qute-UUID
+    Content-Location: http://example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+
+    -----=_qute-UUID
+    MIME-Version: 1.0
+    Content-Location: http://example.com/file
+    Content-Transfer-Encoding: quoted-printable
+
+    file=20content
+    -----=_qute-UUID--
+    """)
+
+
+def test_removing_file_from_mhtml(checker):
+    writer = mhtml.MHTMLWriter(root_content=b"root",
+                               content_location="http://example.com/",
+                               content_type="text/plain")
+    writer.add_file("http://evil.com/", b"file content")
+    writer.remove_file("http://evil.com/")
+    writer.write_to(checker.fp)
+    checker.expect("""
+    Content-Type: multipart/related; boundary="---=_qute-UUID"
+    MIME-Version: 1.0
+
+    -----=_qute-UUID
+    Content-Location: http://example.com/
+    MIME-Version: 1.0
+    Content-Type: text/plain
+    Content-Transfer-Encoding: quoted-printable
+
+    root
+    -----=_qute-UUID--
+    """)
+
+
+@pytest.mark.parametrize("style, expected_urls", [
+    ("@import 'default.css'", ["default.css"]),
+    ('@import "default.css"', ["default.css"]),
+    ("@import \t 'tabbed.css'", ["tabbed.css"]),
+    ("@import url('default.css')", ["default.css"]),
+    ("""body {
+    background: url("/bg-img.png")
+    }""", ["/bg-img.png"]),
+    ("background: url(folder/file.png)", ["folder/file.png"]),
+    ("content: url()", []),
+])
+def test_css_url_scanner(style, expected_urls):
+    expected_urls.sort()
+    urls = mhtml._get_css_imports(style)
+    urls.sort()
+    assert urls == expected_urls

From 366916a8bf602675c5e58c824cc19c323a3d749f Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 20:36:02 +0200
Subject: [PATCH 28/66] Use more specific selectors to filter webelements

---
 qutebrowser/misc/mhtml.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 56cb974a7..9a5d19850 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -201,9 +201,7 @@ class _Downloader():
         )
         # Currently only downloading <link> (stylesheets), <script>
         # (javascript) and <img> (image) elements.
-        elements = (web_frame.findAllElements("link") +
-                    web_frame.findAllElements("script") +
-                    web_frame.findAllElements("img"))
+        elements = web_frame.findAllElements("link, script, img")
 
         for element in elements:
             element = webelem.WebElementWrapper(element)
@@ -226,10 +224,8 @@ class _Downloader():
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
         # Search for references in inline styles
-        for element in web_frame.findAllElements("*"):
+        for element in web_frame.findAllElements("[style]"):
             element = webelem.WebElementWrapper(element)
-            if "style" not in element:
-                continue
             style = element["style"]
             for element_url in _get_css_imports(style):
                 self.fetch_url(web_url.resolved(QUrl(element_url)))

From 9bf9124324b2b1e7c2e9c958bcd1106de9dfdc01 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 20:43:30 +0200
Subject: [PATCH 29/66] Fix mhtml tests, add test for _NoCloseBytesIO

---
 tests/unit/misc/test_mhtml.py | 229 +++++++++++++++++++---------------
 1 file changed, 125 insertions(+), 104 deletions(-)

diff --git a/tests/unit/misc/test_mhtml.py b/tests/unit/misc/test_mhtml.py
index 8f0a59379..271d1fc1b 100644
--- a/tests/unit/misc/test_mhtml.py
+++ b/tests/unit/misc/test_mhtml.py
@@ -50,18 +50,18 @@ def test_quoted_printable_umlauts(checker):
                                content_type="text/plain")
     writer.write_to(checker.fp)
     checker.expect("""
-    Content-Type: multipart/related; boundary="---=_qute-UUID"
-    MIME-Version: 1.0
+        Content-Type: multipart/related; boundary="---=_qute-UUID"
+        MIME-Version: 1.0
 
-    -----=_qute-UUID
-    Content-Location: localhost
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        -----=_qute-UUID
+        Content-Location: localhost
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    Die=20s=FC=DFe=20H=FCndin=20l=E4uft=20in=20die=20H=F6hle=20des=20B=E4ren
-    -----=_qute-UUID--
-    """)
+        Die=20s=FC=DFe=20H=FCndin=20l=E4uft=20in=20die=20H=F6hle=20des=20B=E4ren
+        -----=_qute-UUID--
+        """)
 
 
 @pytest.mark.parametrize("header, value", [
@@ -76,9 +76,9 @@ def test_refuses_non_ascii_header_value(checker, header, value):
     }
     defaults[header] = value
     writer = mhtml.MHTMLWriter(**defaults)
-    with pytest.raises(UnicodeEncodeError) as e:
+    with pytest.raises(UnicodeEncodeError) as excinfo:
         writer.write_to(checker.fp)
-        assert "'ascii' codec can't encode" in str(e)
+        assert "'ascii' codec can't encode" in str(excinfo.value)
 
 
 def test_file_encoded_as_base64(checker):
@@ -91,26 +91,26 @@ def test_file_encoded_as_base64(checker):
                     transfer_encoding=mhtml.E_BASE64)
     writer.write_to(checker.fp)
     checker.expect("""
-    Content-Type: multipart/related; boundary="---=_qute-UUID"
-    MIME-Version: 1.0
+        Content-Type: multipart/related; boundary="---=_qute-UUID"
+        MIME-Version: 1.0
 
-    -----=_qute-UUID
-    Content-Location: http://example.com
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        -----=_qute-UUID
+        Content-Location: http://example.com
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    Image=20file=20attached
-    -----=_qute-UUID
-    Content-Location: http://a.example.com/image.png
-    MIME-Version: 1.0
-    Content-Type: image/png
-    Content-Transfer-Encoding: base64
+        Image=20file=20attached
+        -----=_qute-UUID
+        Content-Location: http://a.example.com/image.png
+        MIME-Version: 1.0
+        Content-Type: image/png
+        Content-Transfer-Encoding: base64
 
-    8J+YgSBpbWFnZSBkYXRh
+        8J+YgSBpbWFnZSBkYXRh
 
-    -----=_qute-UUID--
-    """)
+        -----=_qute-UUID--
+        """)
 
 
 @pytest.mark.parametrize("transfer_encoding", [mhtml.E_BASE64, mhtml.E_QUOPRI],
@@ -138,67 +138,67 @@ def test_files_appear_sorted(checker):
                         transfer_encoding=mhtml.E_QUOPRI)
     writer.write_to(checker.fp)
     checker.expect("""
-    Content-Type: multipart/related; boundary="---=_qute-UUID"
-    MIME-Version: 1.0
+        Content-Type: multipart/related; boundary="---=_qute-UUID"
+        MIME-Version: 1.0
 
-    -----=_qute-UUID
-    Content-Location: http://www.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        -----=_qute-UUID
+        Content-Location: http://www.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    root=20file
-    -----=_qute-UUID
-    Content-Location: http://a.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        root=20file
+        -----=_qute-UUID
+        Content-Location: http://a.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20a
-    -----=_qute-UUID
-    Content-Location: http://b.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        file=20a
+        -----=_qute-UUID
+        Content-Location: http://b.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20b
-    -----=_qute-UUID
-    Content-Location: http://g.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        file=20b
+        -----=_qute-UUID
+        Content-Location: http://g.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20g
-    -----=_qute-UUID
-    Content-Location: http://h.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        file=20g
+        -----=_qute-UUID
+        Content-Location: http://h.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20h
-    -----=_qute-UUID
-    Content-Location: http://i.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        file=20h
+        -----=_qute-UUID
+        Content-Location: http://i.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20i
-    -----=_qute-UUID
-    Content-Location: http://t.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        file=20i
+        -----=_qute-UUID
+        Content-Location: http://t.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20t
-    -----=_qute-UUID
-    Content-Location: http://z.example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        file=20t
+        -----=_qute-UUID
+        Content-Location: http://z.example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20z
-    -----=_qute-UUID--
-    """)
+        file=20z
+        -----=_qute-UUID--
+        """)
 
 
 def test_empty_content_type(checker):
@@ -208,24 +208,24 @@ def test_empty_content_type(checker):
     writer.add_file("http://example.com/file", b"file content")
     writer.write_to(checker.fp)
     checker.expect("""
-    Content-Type: multipart/related; boundary="---=_qute-UUID"
-    MIME-Version: 1.0
+        Content-Type: multipart/related; boundary="---=_qute-UUID"
+        MIME-Version: 1.0
 
-    -----=_qute-UUID
-    Content-Location: http://example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        -----=_qute-UUID
+        Content-Location: http://example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
 
-    -----=_qute-UUID
-    MIME-Version: 1.0
-    Content-Location: http://example.com/file
-    Content-Transfer-Encoding: quoted-printable
+        -----=_qute-UUID
+        MIME-Version: 1.0
+        Content-Location: http://example.com/file
+        Content-Transfer-Encoding: quoted-printable
 
-    file=20content
-    -----=_qute-UUID--
-    """)
+        file=20content
+        -----=_qute-UUID--
+        """)
 
 
 def test_removing_file_from_mhtml(checker):
@@ -236,18 +236,18 @@ def test_removing_file_from_mhtml(checker):
     writer.remove_file("http://evil.com/")
     writer.write_to(checker.fp)
     checker.expect("""
-    Content-Type: multipart/related; boundary="---=_qute-UUID"
-    MIME-Version: 1.0
+        Content-Type: multipart/related; boundary="---=_qute-UUID"
+        MIME-Version: 1.0
 
-    -----=_qute-UUID
-    Content-Location: http://example.com/
-    MIME-Version: 1.0
-    Content-Type: text/plain
-    Content-Transfer-Encoding: quoted-printable
+        -----=_qute-UUID
+        Content-Location: http://example.com/
+        MIME-Version: 1.0
+        Content-Type: text/plain
+        Content-Transfer-Encoding: quoted-printable
 
-    root
-    -----=_qute-UUID--
-    """)
+        root
+        -----=_qute-UUID--
+        """)
 
 
 @pytest.mark.parametrize("style, expected_urls", [
@@ -266,3 +266,24 @@ def test_css_url_scanner(style, expected_urls):
     urls = mhtml._get_css_imports(style)
     urls.sort()
     assert urls == expected_urls
+
+
+def test_noclose_bytesio_fake_close():
+    fp = mhtml._NoCloseBytesIO()
+    fp.write(b'Value')
+    fp.close()
+    assert fp.getvalue() == b'Value'
+    fp.write(b'Eulav')
+    assert fp.getvalue() == b'ValueEulav'
+
+
+def test_noclose_bytesio_actual_close():
+    fp = mhtml._NoCloseBytesIO()
+    fp.write(b'Value')
+    fp.actual_close()
+    with pytest.raises(ValueError) as excinfo:
+        fp.getvalue()
+        assert 'I/O operation on closed file.' == str(excinfo.value)
+    with pytest.raises(ValueError) as excinfo:
+        fp.write(b'Closed')
+        assert 'I/O operation on closed file.' == str(excinfo.value)

From a092ef1fe649399f946d024c08e81653e0a42903 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 21:56:02 +0200
Subject: [PATCH 30/66] String quote style changes

"" for user facing strings
'' for internal strings
except when quotes appear inside a string, to avoid escaping them
---
 qutebrowser/misc/mhtml.py     |  64 ++++++++++----------
 tests/unit/misc/test_mhtml.py | 106 +++++++++++++++++-----------------
 2 files changed, 84 insertions(+), 86 deletions(-)

diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/misc/mhtml.py
index 9a5d19850..ce909f683 100644
--- a/qutebrowser/misc/mhtml.py
+++ b/qutebrowser/misc/mhtml.py
@@ -35,9 +35,9 @@ from qutebrowser.browser import webelem
 from qutebrowser.utils import log, objreg, message
 
 
-_File = collections.namedtuple("_File",
-                               ["content", "content_type", "content_location",
-                                "transfer_encoding"])
+_File = collections.namedtuple('_File',
+                               ['content', 'content_type', 'content_location',
+                                'transfer_encoding'])
 
 
 _CSS_URL_PATTERNS = [re.compile(x) for x in [
@@ -51,9 +51,7 @@ _CSS_URL_PATTERNS = [re.compile(x) for x in [
 
 def _get_css_imports(data):
     """Return all assets that are referenced in the given CSS document.
-
     The returned URLs are relative to the stylesheet's URL.
-
     Args:
         data: The content of the stylesheet to scan as string.
     """
@@ -66,7 +64,7 @@ def _get_css_imports(data):
     return urls
 
 
-MHTMLPolicy = policy.default.clone(linesep="\r\n", max_line_length=0)
+MHTMLPolicy = policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
 E_BASE64 = encoders.encode_base64
@@ -123,8 +121,8 @@ class MHTMLWriter():
         Args:
             fp: The file-object, openend in "wb" mode.
         """
-        msg = multipart.MIMEMultipart("related",
-                                      "---=_qute-{}".format(uuid.uuid4()))
+        msg = multipart.MIMEMultipart('related',
+                                      '---=_qute-{}'.format(uuid.uuid4()))
 
         root = self._create_root_file()
         msg.attach(root)
@@ -146,9 +144,9 @@ class MHTMLWriter():
     def _create_file(self, f):
         """Return the single given file as MIMEMultipart."""
         msg = multipart.MIMEMultipart()
-        msg["Content-Location"] = f.content_location
+        msg['Content-Location'] = f.content_location
         # Get rid of the default type multipart/mixed
-        del msg["Content-Type"]
+        del msg['Content-Type']
         if f.content_type:
             msg.set_type(f.content_type)
         msg.set_payload(f.content)
@@ -192,7 +190,7 @@ class _Downloader():
         web_frame = self.web_view.page().mainFrame()
 
         self.writer = MHTMLWriter(
-            web_frame.toHtml().encode("utf-8"),
+            web_frame.toHtml().encode('utf-8'),
             content_location=web_url.toString(),
             # I've found no way of getting the content type of a QWebView, but
             # since we're using .toHtml, it's probably safe to say that the
@@ -201,32 +199,32 @@ class _Downloader():
         )
         # Currently only downloading <link> (stylesheets), <script>
         # (javascript) and <img> (image) elements.
-        elements = web_frame.findAllElements("link, script, img")
+        elements = web_frame.findAllElements('link, script, img')
 
         for element in elements:
             element = webelem.WebElementWrapper(element)
-            if "src" in element:
-                element_url = element["src"]
-            elif "href" in element:
-                element_url = element["href"]
+            if 'src' in element:
+                element_url = element['src']
+            elif 'href' in element:
+                element_url = element['href']
             else:
                 # Might be a local <script> tag or something else
                 continue
             absolute_url = web_url.resolved(QUrl(element_url))
             self.fetch_url(absolute_url)
 
-        styles = web_frame.findAllElements("style")
+        styles = web_frame.findAllElements('style')
         for style in styles:
             style = webelem.WebElementWrapper(style)
-            if "type" in style and style["type"] != "text/css":
+            if 'type' in style and style['type'] != 'text/css':
                 continue
             for element_url in _get_css_imports(str(style)):
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
         # Search for references in inline styles
-        for element in web_frame.findAllElements("[style]"):
+        for element in web_frame.findAllElements('[style]'):
             element = webelem.WebElementWrapper(element)
-            style = element["style"]
+            style = element['style']
             for element_url in _get_css_imports(style):
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
@@ -243,7 +241,7 @@ class _Downloader():
         Args:
             url: The file to download as QUrl.
         """
-        if url.scheme() == "data":
+        if url.scheme() == 'data':
             return
         # Prevent loading an asset twice
         if url in self.loaded_urls:
@@ -252,8 +250,8 @@ class _Downloader():
 
         log.downloads.debug("loading asset at %s", url)
 
-        download_manager = objreg.get("download-manager", scope="window",
-                                      window="current")
+        download_manager = objreg.get('download-manager', scope='window',
+                                      window='current')
         item = download_manager.get(url, fileobj=_NoCloseBytesIO(),
                                     auto_remove=True)
         self.pending_downloads.add((url, item))
@@ -272,7 +270,7 @@ class _Downloader():
             item: The DownloadItem given by the DownloadManager
         """
         self.pending_downloads.remove((url, item))
-        mime = item.raw_headers.get(b"Content-Type", b"")
+        mime = item.raw_headers.get(b'Content-Type', b'')
 
         # Note that this decoding always works and doesn't produce errors
         # RFC 7230 (https://tools.ietf.org/html/rfc7230) states:
@@ -283,9 +281,9 @@ class _Downloader():
         # Newly defined header fields SHOULD limit their field values to
         # US-ASCII octets.  A recipient SHOULD treat other octets in field
         # content (obs-text) as opaque data.
-        mime = mime.decode("iso-8859-1")
+        mime = mime.decode('iso-8859-1')
 
-        if mime.lower() == "text/css":
+        if mime.lower() == 'text/css':
             # We can't always assume that CSS files are UTF-8, but CSS files
             # shouldn't contain many non-ASCII characters anyway (in most
             # cases). Using "ignore" lets us decode the file even if it's
@@ -293,16 +291,16 @@ class _Downloader():
             # The file written to the MHTML file won't be modified by this
             # decoding, since there we're taking the original bytestream.
             try:
-                css_string = item.fileobj.getvalue().decode("utf-8")
+                css_string = item.fileobj.getvalue().decode('utf-8')
             except UnicodeDecodeError:
                 log.downloads.warning("Invalid UTF-8 data in %s", url)
-                css_string = item.fileobj.getvalue().decode("utf-8", "ignore")
+                css_string = item.fileobj.getvalue().decode('utf-8', 'ignore')
             import_urls = _get_css_imports(css_string)
             for import_url in import_urls:
                 absolute_url = url.resolved(QUrl(import_url))
                 self.fetch_url(absolute_url)
 
-        encode = E_QUOPRI if mime.startswith("text/") else E_BASE64
+        encode = E_QUOPRI if mime.startswith('text/') else E_BASE64
         self.writer.add_file(url.toString(), item.fileobj.getvalue(), mime,
                              encode)
         item.fileobj.actual_close()
@@ -325,7 +323,7 @@ class _Downloader():
             log.downloads.debug("Oops! Download already gone: %s", item)
             return
         item.fileobj.actual_close()
-        self.writer.add_file(url.toString(), b"")
+        self.writer.add_file(url.toString(), b'')
         if self.pending_downloads:
             return
         self.finish_file()
@@ -337,9 +335,9 @@ class _Downloader():
             return
         self._finished = True
         log.downloads.debug("All assets downloaded, ready to finish off!")
-        with open(self.dest, "wb") as file_output:
+        with open(self.dest, 'wb') as file_output:
             self.writer.write_to(file_output)
-        message.info("current", "Page saved as {}".format(self.dest))
+        message.info('current', "Page saved as {}".format(self.dest))
 
     def collect_zombies(self):
         """Collect done downloads and add their data to the MHTML file.
@@ -378,6 +376,6 @@ def start_download(dest):
         dest: The filename where the resulting file should be saved.
     """
     dest = os.path.expanduser(dest)
-    web_view = objreg.get("webview", scope="tab", tab="current")
+    web_view = objreg.get('webview', scope='tab', tab='current')
     loader = _Downloader(web_view, dest)
     loader.run()
diff --git a/tests/unit/misc/test_mhtml.py b/tests/unit/misc/test_mhtml.py
index 271d1fc1b..c3a859488 100644
--- a/tests/unit/misc/test_mhtml.py
+++ b/tests/unit/misc/test_mhtml.py
@@ -28,12 +28,12 @@ class Checker:
         return self.fp.getvalue()
 
     def expect(self, expected):
-        actual = self.value.decode("ascii")
+        actual = self.value.decode('ascii')
         # Make sure there are no stray \r or \n
-        assert re.search(r"\r[^\n]", actual) is None
-        assert re.search(r"[^\r]\n", actual) is None
-        actual = actual.replace("\r\n", "\n")
-        expected = textwrap.dedent(expected).lstrip("\n")
+        assert re.search(r'\r[^\n]', actual) is None
+        assert re.search(r'[^\r]\n', actual) is None
+        actual = actual.replace('\r\n', '\n')
+        expected = textwrap.dedent(expected).lstrip('\n')
         assert expected == actual
 
 
@@ -43,11 +43,11 @@ def checker():
 
 
 def test_quoted_printable_umlauts(checker):
-    content = "Die süße Hündin läuft in die Höhle des Bären"
-    content = content.encode("iso-8859-1")
+    content = 'Die süße Hündin läuft in die Höhle des Bären'
+    content = content.encode('iso-8859-1')
     writer = mhtml.MHTMLWriter(root_content=content,
-                               content_location="localhost",
-                               content_type="text/plain")
+                               content_location='localhost',
+                               content_type='text/plain')
     writer.write_to(checker.fp)
     checker.expect("""
         Content-Type: multipart/related; boundary="---=_qute-UUID"
@@ -64,15 +64,15 @@ def test_quoted_printable_umlauts(checker):
         """)
 
 
-@pytest.mark.parametrize("header, value", [
-    ("content_location", "http://brötli.com"),
-    ("content_type", "text/pläin"),
+@pytest.mark.parametrize('header, value', [
+    ('content_location', 'http://brötli.com'),
+    ('content_type', 'text/pläin'),
 ])
 def test_refuses_non_ascii_header_value(checker, header, value):
     defaults = {
-        "root_content": b"",
-        "content_location": "http://example.com",
-        "content_type": "text/plain",
+        'root_content': b'',
+        'content_location': 'http://example.com',
+        'content_type': 'text/plain',
     }
     defaults[header] = value
     writer = mhtml.MHTMLWriter(**defaults)
@@ -82,12 +82,12 @@ def test_refuses_non_ascii_header_value(checker, header, value):
 
 
 def test_file_encoded_as_base64(checker):
-    content = b"Image file attached"
-    writer = mhtml.MHTMLWriter(root_content=content, content_type="text/plain",
-                               content_location="http://example.com")
-    writer.add_file(location="http://a.example.com/image.png",
-                    content="\U0001F601 image data".encode("utf-8"),
-                    content_type="image/png",
+    content = b'Image file attached'
+    writer = mhtml.MHTMLWriter(root_content=content, content_type='text/plain',
+                               content_location='http://example.com')
+    writer.add_file(location='http://a.example.com/image.png',
+                    content='\U0001F601 image data'.encode('utf-8'),
+                    content_type='image/png',
                     transfer_encoding=mhtml.E_BASE64)
     writer.write_to(checker.fp)
     checker.expect("""
@@ -113,28 +113,28 @@ def test_file_encoded_as_base64(checker):
         """)
 
 
-@pytest.mark.parametrize("transfer_encoding", [mhtml.E_BASE64, mhtml.E_QUOPRI],
-                         ids=["base64", "quoted-printable"])
+@pytest.mark.parametrize('transfer_encoding', [mhtml.E_BASE64, mhtml.E_QUOPRI],
+                         ids=['base64', 'quoted-printable'])
 def test_payload_lines_wrap(checker, transfer_encoding):
-    payload = b"1234567890" * 10
-    writer = mhtml.MHTMLWriter(root_content=b"", content_type="text/plain",
-                               content_location="http://example.com")
-    writer.add_file(location="http://example.com/payload", content=payload,
-                    content_type="text/plain",
+    payload = b'1234567890' * 10
+    writer = mhtml.MHTMLWriter(root_content=b'', content_type='text/plain',
+                               content_location='http://example.com')
+    writer.add_file(location='http://example.com/payload', content=payload,
+                    content_type='text/plain',
                     transfer_encoding=transfer_encoding)
     writer.write_to(checker.fp)
-    for line in checker.value.split(b"\r\n"):
+    for line in checker.value.split(b'\r\n'):
         assert len(line) < 77
 
 
 def test_files_appear_sorted(checker):
-    writer = mhtml.MHTMLWriter(root_content=b"root file",
-                               content_type="text/plain",
-                               content_location="http://www.example.com/")
-    for subdomain in "ahgbizt":
-        writer.add_file(location="http://{}.example.com/".format(subdomain),
-                        content="file {}".format(subdomain).encode("utf-8"),
-                        content_type="text/plain",
+    writer = mhtml.MHTMLWriter(root_content=b'root file',
+                               content_type='text/plain',
+                               content_location='http://www.example.com/')
+    for subdomain in 'ahgbizt':
+        writer.add_file(location='http://{}.example.com/'.format(subdomain),
+                        content='file {}'.format(subdomain).encode('utf-8'),
+                        content_type='text/plain',
                         transfer_encoding=mhtml.E_QUOPRI)
     writer.write_to(checker.fp)
     checker.expect("""
@@ -202,10 +202,10 @@ def test_files_appear_sorted(checker):
 
 
 def test_empty_content_type(checker):
-    writer = mhtml.MHTMLWriter(root_content=b"",
-                               content_location="http://example.com/",
-                               content_type="text/plain")
-    writer.add_file("http://example.com/file", b"file content")
+    writer = mhtml.MHTMLWriter(root_content=b'',
+                               content_location='http://example.com/',
+                               content_type='text/plain')
+    writer.add_file('http://example.com/file', b'file content')
     writer.write_to(checker.fp)
     checker.expect("""
         Content-Type: multipart/related; boundary="---=_qute-UUID"
@@ -229,11 +229,11 @@ def test_empty_content_type(checker):
 
 
 def test_removing_file_from_mhtml(checker):
-    writer = mhtml.MHTMLWriter(root_content=b"root",
-                               content_location="http://example.com/",
-                               content_type="text/plain")
-    writer.add_file("http://evil.com/", b"file content")
-    writer.remove_file("http://evil.com/")
+    writer = mhtml.MHTMLWriter(root_content=b'root',
+                               content_location='http://example.com/',
+                               content_type='text/plain')
+    writer.add_file('http://evil.com/', b'file content')
+    writer.remove_file('http://evil.com/')
     writer.write_to(checker.fp)
     checker.expect("""
         Content-Type: multipart/related; boundary="---=_qute-UUID"
@@ -250,16 +250,16 @@ def test_removing_file_from_mhtml(checker):
         """)
 
 
-@pytest.mark.parametrize("style, expected_urls", [
-    ("@import 'default.css'", ["default.css"]),
-    ('@import "default.css"', ["default.css"]),
-    ("@import \t 'tabbed.css'", ["tabbed.css"]),
-    ("@import url('default.css')", ["default.css"]),
+@pytest.mark.parametrize('style, expected_urls', [
+    ("@import 'default.css'", ['default.css']),
+    ('@import "default.css"', ['default.css']),
+    ("@import \t 'tabbed.css'", ['tabbed.css']),
+    ("@import url('default.css')", ['default.css']),
     ("""body {
     background: url("/bg-img.png")
-    }""", ["/bg-img.png"]),
-    ("background: url(folder/file.png)", ["folder/file.png"]),
-    ("content: url()", []),
+    }""", ['/bg-img.png']),
+    ('background: url(folder/file.png)', ['folder/file.png']),
+    ('content: url()', []),
 ])
 def test_css_url_scanner(style, expected_urls):
     expected_urls.sort()

From 2eeace1c2c5037547862b1eb0d6d9a9878957037 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 22:06:34 +0200
Subject: [PATCH 31/66] Move misc.mhtml to browser.mhtml

---
 qutebrowser/browser/commands.py            | 4 ++--
 qutebrowser/{misc => browser}/mhtml.py     | 0
 tests/unit/{misc => browser}/test_mhtml.py | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename qutebrowser/{misc => browser}/mhtml.py (100%)
 rename tests/unit/{misc => browser}/test_mhtml.py (100%)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index f1b3f4661..11b4f9050 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -37,12 +37,12 @@ import pygments.formatters
 
 from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
 from qutebrowser.config import config, configexc
-from qutebrowser.browser import webelem, inspector, urlmarks, downloads
+from qutebrowser.browser import webelem, inspector, urlmarks, downloads, mhtml
 from qutebrowser.keyinput import modeman
 from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
                                objreg, utils)
 from qutebrowser.utils.usertypes import KeyMode
-from qutebrowser.misc import editor, guiprocess, mhtml
+from qutebrowser.misc import editor, guiprocess
 
 
 class CommandDispatcher:
diff --git a/qutebrowser/misc/mhtml.py b/qutebrowser/browser/mhtml.py
similarity index 100%
rename from qutebrowser/misc/mhtml.py
rename to qutebrowser/browser/mhtml.py
diff --git a/tests/unit/misc/test_mhtml.py b/tests/unit/browser/test_mhtml.py
similarity index 100%
rename from tests/unit/misc/test_mhtml.py
rename to tests/unit/browser/test_mhtml.py

From b05a0d191de7021c30e1061bdceb1aa95d3d9432 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 24 Sep 2015 22:17:47 +0200
Subject: [PATCH 32/66] Fix module path in test_mhtml

Also fix docstring for _get_css_imports
---
 qutebrowser/browser/mhtml.py     | 2 ++
 tests/unit/browser/test_mhtml.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index ce909f683..0fac55003 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -51,7 +51,9 @@ _CSS_URL_PATTERNS = [re.compile(x) for x in [
 
 def _get_css_imports(data):
     """Return all assets that are referenced in the given CSS document.
+
     The returned URLs are relative to the stylesheet's URL.
+
     Args:
         data: The content of the stylesheet to scan as string.
     """
diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index c3a859488..106cb3fd7 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -5,7 +5,7 @@ import re
 
 import pytest
 
-from qutebrowser.misc import mhtml
+from qutebrowser.browser import mhtml
 
 @pytest.fixture(autouse=True)
 def patch_uuid(monkeypatch):

From 749b1c02ccdaacc4416cd267eb6a0375cdbb3d41 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Fri, 25 Sep 2015 11:48:43 +0200
Subject: [PATCH 33/66] Style changes for mhtml and test_mhtml

---
 qutebrowser/browser/mhtml.py     | 21 +++++++++--------
 tests/unit/browser/test_mhtml.py | 39 +++++++++++++++++---------------
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 0fac55003..cc394b21b 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -23,11 +23,12 @@ import functools
 import io
 import os
 import re
-
 import collections
 import uuid
-from email import policy, generator, encoders
-from email.mime import multipart
+import email.policy
+import email.generator
+import email.encoders
+import email.mime.multipart
 
 from PyQt5.QtCore import QUrl
 
@@ -66,13 +67,13 @@ def _get_css_imports(data):
     return urls
 
 
-MHTMLPolicy = policy.default.clone(linesep='\r\n', max_line_length=0)
+MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
-E_BASE64 = encoders.encode_base64
+E_BASE64 = email.encoders.encode_base64
 """Encode the file using base64 encoding"""
 
-E_QUOPRI = encoders.encode_quopri
+E_QUOPRI = email.encoders.encode_quopri
 """Encode the file using MIME quoted-printable encoding."""
 
 
@@ -123,8 +124,8 @@ class MHTMLWriter():
         Args:
             fp: The file-object, openend in "wb" mode.
         """
-        msg = multipart.MIMEMultipart('related',
-                                      '---=_qute-{}'.format(uuid.uuid4()))
+        msg = email.mime.multipart.MIMEMultipart(
+            'related', '---=_qute-{}'.format(uuid.uuid4()))
 
         root = self._create_root_file()
         msg.attach(root)
@@ -132,7 +133,7 @@ class MHTMLWriter():
         for _, file_data in sorted(self._files.items()):
             msg.attach(self._create_file(file_data))
 
-        gen = generator.BytesGenerator(fp, policy=MHTMLPolicy)
+        gen = email.generator.BytesGenerator(fp, policy=MHTMLPolicy)
         gen.flatten(msg)
 
     def _create_root_file(self):
@@ -145,7 +146,7 @@ class MHTMLWriter():
 
     def _create_file(self, f):
         """Return the single given file as MIMEMultipart."""
-        msg = multipart.MIMEMultipart()
+        msg = email.mime.multipart.MIMEMultipart()
         msg['Content-Location'] = f.content_location
         # Get rid of the default type multipart/mixed
         del msg['Content-Type']
diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 106cb3fd7..37d1131b3 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -78,7 +78,7 @@ def test_refuses_non_ascii_header_value(checker, header, value):
     writer = mhtml.MHTMLWriter(**defaults)
     with pytest.raises(UnicodeEncodeError) as excinfo:
         writer.write_to(checker.fp)
-        assert "'ascii' codec can't encode" in str(excinfo.value)
+    assert "'ascii' codec can't encode" in str(excinfo.value)
 
 
 def test_file_encoded_as_base64(checker):
@@ -268,22 +268,25 @@ def test_css_url_scanner(style, expected_urls):
     assert urls == expected_urls
 
 
-def test_noclose_bytesio_fake_close():
-    fp = mhtml._NoCloseBytesIO()
-    fp.write(b'Value')
-    fp.close()
-    assert fp.getvalue() == b'Value'
-    fp.write(b'Eulav')
-    assert fp.getvalue() == b'ValueEulav'
+class TestNoCloseBytesIO:
+    # WORKAROUND for https://bitbucket.org/logilab/pylint/issues/540/
+    # pylint: disable=no-member
 
+    def test_fake_close(self):
+        fp = mhtml._NoCloseBytesIO()
+        fp.write(b'Value')
+        fp.close()
+        assert fp.getvalue() == b'Value'
+        fp.write(b'Eulav')
+        assert fp.getvalue() == b'ValueEulav'
 
-def test_noclose_bytesio_actual_close():
-    fp = mhtml._NoCloseBytesIO()
-    fp.write(b'Value')
-    fp.actual_close()
-    with pytest.raises(ValueError) as excinfo:
-        fp.getvalue()
-        assert 'I/O operation on closed file.' == str(excinfo.value)
-    with pytest.raises(ValueError) as excinfo:
-        fp.write(b'Closed')
-        assert 'I/O operation on closed file.' == str(excinfo.value)
+    def test_actual_close(self):
+        fp = mhtml._NoCloseBytesIO()
+        fp.write(b'Value')
+        fp.actual_close()
+        with pytest.raises(ValueError) as excinfo:
+            fp.getvalue()
+        assert str(excinfo.value) == 'I/O operation on closed file.'
+        with pytest.raises(ValueError) as excinfo:
+            fp.write(b'Closed')
+        assert str(excinfo.value) == 'I/O operation on closed file.'

From 420c0873734dce3594553b62917e2b7c0b439a7f Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Fri, 25 Sep 2015 13:09:39 +0200
Subject: [PATCH 34/66] use cssutils

---
 qutebrowser/browser/mhtml.py     | 49 ++++++++++++++++++++++++++++++--
 requirements.txt                 |  1 +
 tests/unit/browser/test_mhtml.py | 26 ++++++++++-------
 3 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index cc394b21b..7e0422eaf 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -35,6 +35,10 @@ from PyQt5.QtCore import QUrl
 from qutebrowser.browser import webelem
 from qutebrowser.utils import log, objreg, message
 
+try:
+    import cssutils
+except ImportError:
+    cssutils = None
 
 _File = collections.namedtuple('_File',
                                ['content', 'content_type', 'content_location',
@@ -50,7 +54,7 @@ _CSS_URL_PATTERNS = [re.compile(x) for x in [
 ]]
 
 
-def _get_css_imports(data):
+def _get_css_imports_regex(data):
     """Return all assets that are referenced in the given CSS document.
 
     The returned URLs are relative to the stylesheet's URL.
@@ -67,6 +71,47 @@ def _get_css_imports(data):
     return urls
 
 
+def _get_css_imports_cssutils(data, inline=False):
+    """Return all assets that are referenced in the given CSS document.
+
+    The returned URLs are relative to the stylesheet's URL.
+
+    Args:
+        data: The content of the stylesheet to scan as string.
+        inline: True if the argument is a inline HTML style attribute.
+    """
+    parser = cssutils.CSSParser(fetcher=lambda url: (None, ""), validate=False)
+    if not inline:
+        sheet = parser.parseString(data)
+        return list(cssutils.getUrls(sheet))
+    else:
+        urls = []
+        declaration = parser.parseStyle(data)
+        # prop = background, color, margin, ...
+        for prop in declaration:
+            # value = red, 10px, url(foobar), ...
+            for value in prop.propertyValue:
+                if isinstance(value, cssutils.css.URIValue):
+                    if value.uri:
+                        urls.append(value.uri)
+        return urls
+
+
+def _get_css_imports(data, inline=False):
+    """Return all assets that are referenced in the given CSS document.
+
+    The returned URLs are relative to the stylesheet's URL.
+
+    Args:
+        data: The content of the stylesheet to scan as string.
+        inline: True if the argument is a inline HTML style attribute.
+    """
+    if cssutils is None:
+        return _get_css_imports_regex(data)
+    else:
+        return _get_css_imports_cssutils(data, inline)
+
+
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
@@ -228,7 +273,7 @@ class _Downloader():
         for element in web_frame.findAllElements('[style]'):
             element = webelem.WebElementWrapper(element)
             style = element['style']
-            for element_url in _get_css_imports(style):
+            for element_url in _get_css_imports(style, inline=True):
                 self.fetch_url(web_url.resolved(QUrl(element_url)))
 
         # Shortcut if no assets need to be downloaded, otherwise the file would
diff --git a/requirements.txt b/requirements.txt
index fd5f81a0b..9e07256bd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ pyPEG2==2.15.2
 PyYAML==3.11
 colorama==0.3.3
 colorlog==2.6.0
+cssutils==1.0
diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 37d1131b3..71c6ca2d6 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -250,20 +250,26 @@ def test_removing_file_from_mhtml(checker):
         """)
 
 
-@pytest.mark.parametrize('style, expected_urls', [
-    ("@import 'default.css'", ['default.css']),
-    ('@import "default.css"', ['default.css']),
-    ("@import \t 'tabbed.css'", ['tabbed.css']),
-    ("@import url('default.css')", ['default.css']),
-    ("""body {
+@pytest.mark.parametrize('has_cssutils', [True, False])
+@pytest.mark.parametrize('inline, style, expected_urls', [
+    (False, "@import 'default.css'", ['default.css']),
+    (False, '@import "default.css"', ['default.css']),
+    (False, "@import \t 'tabbed.css'", ['tabbed.css']),
+    (False, "@import url('default.css')", ['default.css']),
+    (False, """body {
     background: url("/bg-img.png")
     }""", ['/bg-img.png']),
-    ('background: url(folder/file.png)', ['folder/file.png']),
-    ('content: url()', []),
+    (True, 'background: url(folder/file.png) no-repeat', ['folder/file.png']),
+    (True, 'content: url()', []),
 ])
-def test_css_url_scanner(style, expected_urls):
+def test_css_url_scanner(monkeypatch, has_cssutils, inline, style,
+                         expected_urls):
+    if has_cssutils:
+        assert mhtml.cssutils is not None
+    else:
+        monkeypatch.setattr('qutebrowser.browser.mhtml.cssutils', None)
     expected_urls.sort()
-    urls = mhtml._get_css_imports(style)
+    urls = mhtml._get_css_imports(style, inline=inline)
     urls.sort()
     assert urls == expected_urls
 

From 6601df14a36ec5d05f20684c8eff8de282ea8bdd Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Fri, 2 Oct 2015 13:50:19 +0200
Subject: [PATCH 35/66] mhtml: ask before overwriting dest

---
 qutebrowser/browser/commands.py |  4 ++--
 qutebrowser/browser/mhtml.py    | 28 ++++++++++++++++++++++++++--
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 11b4f9050..a2cfff718 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1171,12 +1171,12 @@ class CommandDispatcher:
             q.mode = usertypes.PromptMode.text
             q.completed.connect(q.deleteLater)
             q.default = downloads.path_suggestion(suggested_fn)
-            q.answered.connect(mhtml.start_download)
+            q.answered.connect(mhtml.start_download_checked)
             message_bridge = objreg.get("message-bridge", scope="window",
                                         window=self._win_id)
             message_bridge.ask(q, blocking=False)
         else:
-            mhtml.start_download(dest)
+            mhtml.start_download_checked(dest)
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        deprecated="Use :download instead.")
diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 7e0422eaf..50c207487 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -33,7 +33,7 @@ import email.mime.multipart
 from PyQt5.QtCore import QUrl
 
 from qutebrowser.browser import webelem
-from qutebrowser.utils import log, objreg, message
+from qutebrowser.utils import log, objreg, message, usertypes
 
 try:
     import cssutils
@@ -80,7 +80,10 @@ def _get_css_imports_cssutils(data, inline=False):
         data: The content of the stylesheet to scan as string.
         inline: True if the argument is a inline HTML style attribute.
     """
-    parser = cssutils.CSSParser(fetcher=lambda url: (None, ""), validate=False)
+    # We don't care about invalid CSS data, this will only litter the log
+    # output with CSS errors
+    parser = cssutils.CSSParser(loglevel=100,
+                                fetcher=lambda url: (None, ""), validate=False)
     if not inline:
         sheet = parser.parseString(data)
         return list(cssutils.getUrls(sheet))
@@ -420,6 +423,8 @@ class _NoCloseBytesIO(io.BytesIO):  # pylint: disable=no-init
 def start_download(dest):
     """Start downloading the current page and all assets to a MHTML file.
 
+    This will overwrite dest if it already exists.
+
     Args:
         dest: The filename where the resulting file should be saved.
     """
@@ -427,3 +432,22 @@ def start_download(dest):
     web_view = objreg.get('webview', scope='tab', tab='current')
     loader = _Downloader(web_view, dest)
     loader.run()
+
+
+def start_download_checked(dest):
+    """First check if dest is already a file, then start the download.
+
+    Args:
+        dest: The filename where the resulting file should be saved.
+    """
+    if not os.path.isfile(dest):
+        start_download(dest)
+        return
+
+    q = usertypes.Question()
+    q.mode = usertypes.PromptMode.yesno
+    q.text = "{} exists. Overwrite?".format(dest)
+    q.completed.connect(q.deleteLater)
+    q.answered_yes.connect(functools.partial(start_download, dest))
+    message_bridge = objreg.get('message-bridge', scope='window', window='current')
+    message_bridge.ask(q, blocking=False)

From 706b8c660077c240ca4ab67070b5641c4d0346d0 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Fri, 2 Oct 2015 19:12:30 +0200
Subject: [PATCH 36/66] Shorten line

---
 qutebrowser/browser/mhtml.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 50c207487..184465b89 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -449,5 +449,6 @@ def start_download_checked(dest):
     q.text = "{} exists. Overwrite?".format(dest)
     q.completed.connect(q.deleteLater)
     q.answered_yes.connect(functools.partial(start_download, dest))
-    message_bridge = objreg.get('message-bridge', scope='window', window='current')
+    message_bridge = objreg.get('message-bridge', scope='window',
+                                window='current')
     message_bridge.ask(q, blocking=False)

From ce1a99cc7cfa9f1fc39804848d97f18c039d175d Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 5 Oct 2015 11:52:04 +0200
Subject: [PATCH 37/66] Remove cssutils from mhtml.py

---
 qutebrowser/browser/mhtml.py     | 66 ++++++--------------------------
 requirements.txt                 |  1 -
 tests/unit/browser/test_mhtml.py |  8 +---
 3 files changed, 12 insertions(+), 63 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 184465b89..54eaaa4f0 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -35,10 +35,6 @@ from PyQt5.QtCore import QUrl
 from qutebrowser.browser import webelem
 from qutebrowser.utils import log, objreg, message, usertypes
 
-try:
-    import cssutils
-except ImportError:
-    cssutils = None
 
 _File = collections.namedtuple('_File',
                                ['content', 'content_type', 'content_location',
@@ -54,52 +50,6 @@ _CSS_URL_PATTERNS = [re.compile(x) for x in [
 ]]
 
 
-def _get_css_imports_regex(data):
-    """Return all assets that are referenced in the given CSS document.
-
-    The returned URLs are relative to the stylesheet's URL.
-
-    Args:
-        data: The content of the stylesheet to scan as string.
-    """
-    urls = []
-    for pattern in _CSS_URL_PATTERNS:
-        for match in pattern.finditer(data):
-            url = match.group("url")
-            if url:
-                urls.append(url)
-    return urls
-
-
-def _get_css_imports_cssutils(data, inline=False):
-    """Return all assets that are referenced in the given CSS document.
-
-    The returned URLs are relative to the stylesheet's URL.
-
-    Args:
-        data: The content of the stylesheet to scan as string.
-        inline: True if the argument is a inline HTML style attribute.
-    """
-    # We don't care about invalid CSS data, this will only litter the log
-    # output with CSS errors
-    parser = cssutils.CSSParser(loglevel=100,
-                                fetcher=lambda url: (None, ""), validate=False)
-    if not inline:
-        sheet = parser.parseString(data)
-        return list(cssutils.getUrls(sheet))
-    else:
-        urls = []
-        declaration = parser.parseStyle(data)
-        # prop = background, color, margin, ...
-        for prop in declaration:
-            # value = red, 10px, url(foobar), ...
-            for value in prop.propertyValue:
-                if isinstance(value, cssutils.css.URIValue):
-                    if value.uri:
-                        urls.append(value.uri)
-        return urls
-
-
 def _get_css_imports(data, inline=False):
     """Return all assets that are referenced in the given CSS document.
 
@@ -107,12 +57,18 @@ def _get_css_imports(data, inline=False):
 
     Args:
         data: The content of the stylesheet to scan as string.
-        inline: True if the argument is a inline HTML style attribute.
+        inline: True if data is a HTML inline style (style="...").
     """
-    if cssutils is None:
-        return _get_css_imports_regex(data)
-    else:
-        return _get_css_imports_cssutils(data, inline)
+    # We keep the inline argument to stay consistent with the cssutils
+    # interface, in case we reintroduce cssutils.
+    # pylint: disable=unused-argument
+    urls = []
+    for pattern in _CSS_URL_PATTERNS:
+        for match in pattern.finditer(data):
+            url = match.group("url")
+            if url:
+                urls.append(url)
+    return urls
 
 
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
diff --git a/requirements.txt b/requirements.txt
index 9e07256bd..fd5f81a0b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,3 @@ pyPEG2==2.15.2
 PyYAML==3.11
 colorama==0.3.3
 colorlog==2.6.0
-cssutils==1.0
diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 71c6ca2d6..06276ae6a 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -250,7 +250,6 @@ def test_removing_file_from_mhtml(checker):
         """)
 
 
-@pytest.mark.parametrize('has_cssutils', [True, False])
 @pytest.mark.parametrize('inline, style, expected_urls', [
     (False, "@import 'default.css'", ['default.css']),
     (False, '@import "default.css"', ['default.css']),
@@ -262,12 +261,7 @@ def test_removing_file_from_mhtml(checker):
     (True, 'background: url(folder/file.png) no-repeat', ['folder/file.png']),
     (True, 'content: url()', []),
 ])
-def test_css_url_scanner(monkeypatch, has_cssutils, inline, style,
-                         expected_urls):
-    if has_cssutils:
-        assert mhtml.cssutils is not None
-    else:
-        monkeypatch.setattr('qutebrowser.browser.mhtml.cssutils', None)
+def test_css_url_scanner(monkeypatch, inline, style, expected_urls):
     expected_urls.sort()
     urls = mhtml._get_css_imports(style, inline=inline)
     urls.sort()

From 957d68c477c65e3422219d785aee35613876d365 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 11:21:47 +0200
Subject: [PATCH 38/66] Revert "Remove cssutils from mhtml.py"

This reverts commit 22a0f0952704d284846ab2572790d99a85515c57.
---
 qutebrowser/browser/mhtml.py     | 54 +++++++++++++++++++++++++++++---
 requirements.txt                 |  1 +
 tests/unit/browser/test_mhtml.py |  8 ++++-
 3 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 54eaaa4f0..184465b89 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -35,6 +35,10 @@ from PyQt5.QtCore import QUrl
 from qutebrowser.browser import webelem
 from qutebrowser.utils import log, objreg, message, usertypes
 
+try:
+    import cssutils
+except ImportError:
+    cssutils = None
 
 _File = collections.namedtuple('_File',
                                ['content', 'content_type', 'content_location',
@@ -50,18 +54,14 @@ _CSS_URL_PATTERNS = [re.compile(x) for x in [
 ]]
 
 
-def _get_css_imports(data, inline=False):
+def _get_css_imports_regex(data):
     """Return all assets that are referenced in the given CSS document.
 
     The returned URLs are relative to the stylesheet's URL.
 
     Args:
         data: The content of the stylesheet to scan as string.
-        inline: True if data is a HTML inline style (style="...").
     """
-    # We keep the inline argument to stay consistent with the cssutils
-    # interface, in case we reintroduce cssutils.
-    # pylint: disable=unused-argument
     urls = []
     for pattern in _CSS_URL_PATTERNS:
         for match in pattern.finditer(data):
@@ -71,6 +71,50 @@ def _get_css_imports(data, inline=False):
     return urls
 
 
+def _get_css_imports_cssutils(data, inline=False):
+    """Return all assets that are referenced in the given CSS document.
+
+    The returned URLs are relative to the stylesheet's URL.
+
+    Args:
+        data: The content of the stylesheet to scan as string.
+        inline: True if the argument is a inline HTML style attribute.
+    """
+    # We don't care about invalid CSS data, this will only litter the log
+    # output with CSS errors
+    parser = cssutils.CSSParser(loglevel=100,
+                                fetcher=lambda url: (None, ""), validate=False)
+    if not inline:
+        sheet = parser.parseString(data)
+        return list(cssutils.getUrls(sheet))
+    else:
+        urls = []
+        declaration = parser.parseStyle(data)
+        # prop = background, color, margin, ...
+        for prop in declaration:
+            # value = red, 10px, url(foobar), ...
+            for value in prop.propertyValue:
+                if isinstance(value, cssutils.css.URIValue):
+                    if value.uri:
+                        urls.append(value.uri)
+        return urls
+
+
+def _get_css_imports(data, inline=False):
+    """Return all assets that are referenced in the given CSS document.
+
+    The returned URLs are relative to the stylesheet's URL.
+
+    Args:
+        data: The content of the stylesheet to scan as string.
+        inline: True if the argument is a inline HTML style attribute.
+    """
+    if cssutils is None:
+        return _get_css_imports_regex(data)
+    else:
+        return _get_css_imports_cssutils(data, inline)
+
+
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
diff --git a/requirements.txt b/requirements.txt
index fd5f81a0b..9e07256bd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ pyPEG2==2.15.2
 PyYAML==3.11
 colorama==0.3.3
 colorlog==2.6.0
+cssutils==1.0
diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 06276ae6a..71c6ca2d6 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -250,6 +250,7 @@ def test_removing_file_from_mhtml(checker):
         """)
 
 
+@pytest.mark.parametrize('has_cssutils', [True, False])
 @pytest.mark.parametrize('inline, style, expected_urls', [
     (False, "@import 'default.css'", ['default.css']),
     (False, '@import "default.css"', ['default.css']),
@@ -261,7 +262,12 @@ def test_removing_file_from_mhtml(checker):
     (True, 'background: url(folder/file.png) no-repeat', ['folder/file.png']),
     (True, 'content: url()', []),
 ])
-def test_css_url_scanner(monkeypatch, inline, style, expected_urls):
+def test_css_url_scanner(monkeypatch, has_cssutils, inline, style,
+                         expected_urls):
+    if has_cssutils:
+        assert mhtml.cssutils is not None
+    else:
+        monkeypatch.setattr('qutebrowser.browser.mhtml.cssutils', None)
     expected_urls.sort()
     urls = mhtml._get_css_imports(style, inline=inline)
     urls.sort()

From 1902e4858f1a6bd03d8a9edc7ae55642db66636f Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 11:24:25 +0200
Subject: [PATCH 39/66] Also catch re.error on cssutils import

cssutils 1.0 and earlier are broken on Python 3.5 due to a bad regex
escape.
---
 qutebrowser/browser/mhtml.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 184465b89..0475651e4 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -37,7 +37,10 @@ from qutebrowser.utils import log, objreg, message, usertypes
 
 try:
     import cssutils
-except ImportError:
+except (ImportError, re.error):
+    # Catching re.error because cssutils in earlier releases (<= 1.0) is broken
+    # on Python 3.5
+    # See https://bitbucket.org/cthedot/cssutils/issues/52
     cssutils = None
 
 _File = collections.namedtuple('_File',

From 919365dfa1693405a49b0e32a246018e043a1cbc Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 11:34:59 +0200
Subject: [PATCH 40/66] Remove dead code mhtml.py:MHTMLWriter:remove_file

---
 qutebrowser/browser/mhtml.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 0475651e4..da598543d 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -161,14 +161,6 @@ class MHTMLWriter():
             content_location=location, transfer_encoding=transfer_encoding,
         )
 
-    def remove_file(self, location):
-        """Remove a file.
-
-        Args:
-            location: The URL that identifies the file.
-        """
-        del self._files[location]
-
     def write_to(self, fp):
         """Output the MHTML file to the given file-like object.
 

From dab0db30a5ad3493354b8040689f7f34137c6450 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 11:46:05 +0200
Subject: [PATCH 41/66] Remove tests for remove_file from test_mhtml.py

---
 tests/unit/browser/test_mhtml.py | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 71c6ca2d6..643f0ded4 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -228,28 +228,6 @@ def test_empty_content_type(checker):
         """)
 
 
-def test_removing_file_from_mhtml(checker):
-    writer = mhtml.MHTMLWriter(root_content=b'root',
-                               content_location='http://example.com/',
-                               content_type='text/plain')
-    writer.add_file('http://evil.com/', b'file content')
-    writer.remove_file('http://evil.com/')
-    writer.write_to(checker.fp)
-    checker.expect("""
-        Content-Type: multipart/related; boundary="---=_qute-UUID"
-        MIME-Version: 1.0
-
-        -----=_qute-UUID
-        Content-Location: http://example.com/
-        MIME-Version: 1.0
-        Content-Type: text/plain
-        Content-Transfer-Encoding: quoted-printable
-
-        root
-        -----=_qute-UUID--
-        """)
-
-
 @pytest.mark.parametrize('has_cssutils', [True, False])
 @pytest.mark.parametrize('inline, style, expected_urls', [
     (False, "@import 'default.css'", ['default.css']),

From b027e6af1ba65a4416456cb10e25545b589b8d12 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 11:49:17 +0200
Subject: [PATCH 42/66] Mark cssutils tests as xfail on Python >= 3.5

---
 tests/unit/browser/test_mhtml.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 643f0ded4..f3375fd66 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -2,6 +2,7 @@
 import io
 import textwrap
 import re
+import sys
 
 import pytest
 
@@ -228,7 +229,12 @@ def test_empty_content_type(checker):
         """)
 
 
-@pytest.mark.parametrize('has_cssutils', [True, False])
+@pytest.mark.parametrize('has_cssutils', [
+    # TODO: remove xfail once new cssutils is released
+    pytest.mark.xfail(sys.version_info >= (3,5), reason='https://'
+                      'bitbucket.org/cthedot/cssutils/issues/52')(True),
+    False,
+])
 @pytest.mark.parametrize('inline, style, expected_urls', [
     (False, "@import 'default.css'", ['default.css']),
     (False, '@import "default.css"', ['default.css']),

From dd8ff860f46c1ce12f332d153a34aa1960bd8447 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 12:08:54 +0200
Subject: [PATCH 43/66] Fix lint

---
 tests/unit/browser/test_mhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index f3375fd66..3ac115ca5 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -231,7 +231,7 @@ def test_empty_content_type(checker):
 
 @pytest.mark.parametrize('has_cssutils', [
     # TODO: remove xfail once new cssutils is released
-    pytest.mark.xfail(sys.version_info >= (3,5), reason='https://'
+    pytest.mark.xfail(sys.version_info >= (3, 5), reason='https://'
                       'bitbucket.org/cthedot/cssutils/issues/52')(True),
     False,
 ])

From b17d74452ffffacfb0f545301c771f36be98ee5f Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 12:12:02 +0200
Subject: [PATCH 44/66] Expand $HOME before checking if file exists

Otherwise we might accidentally overwrite a file.
---
 qutebrowser/browser/mhtml.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index da598543d..56fc600e6 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -435,7 +435,11 @@ def start_download_checked(dest):
     Args:
         dest: The filename where the resulting file should be saved.
     """
-    if not os.path.isfile(dest):
+    # start_download will call os.path.expanduser on dest too, so no need to
+    # overwrite dest. We just want to make sure that we're checking
+    # the right path here. This also means that the user question will show the
+    # original path, not the expanded.
+    if not os.path.isfile(os.path.expanduser(dest)):
         start_download(dest)
         return
 

From a898fd21d176807923947622b757ed893ee65e62 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 13:07:59 +0200
Subject: [PATCH 45/66] Update docs

---
 doc/help/commands.asciidoc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc
index f47bdabd3..f54bd6010 100644
--- a/doc/help/commands.asciidoc
+++ b/doc/help/commands.asciidoc
@@ -19,6 +19,7 @@
 |<<download-open,download-open>>|Open the last/[count]th download.
 |<<download-remove,download-remove>>|Remove the last/[count]th download from the list.
 |<<download-retry,download-retry>>|Retry the first failed/[count]th download.
+|<<download-whole,download-whole>>|Download the current page as a MHTML file, including all assets.
 |<<forward,forward>>|Go forward in the history of the current tab.
 |<<fullscreen,fullscreen>>|Toggle fullscreen mode.
 |<<help,help>>|Show help about a command or setting.
@@ -197,6 +198,15 @@ Retry the first failed/[count]th download.
 ==== count
 The index of the download to cancel.
 
+[[download-whole]]
+=== download-whole
+Syntax: +:download-whole ['dest']+
+
+Download the current page as a MHTML file, including all assets.
+
+==== positional arguments
+* +'dest'+: The file path to write the download to.
+
 [[forward]]
 === forward
 Syntax: +:forward [*--tab*] [*--bg*] [*--window*]+

From 8cf0af004fd8f3c2e9f7e8e23a1c2bc6107ebe86 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 23:13:17 +0200
Subject: [PATCH 46/66] Deprecate :download [url] [dest], add --dest param

:download --dest [dest] [url] is the new syntax.
---
 doc/help/commands.asciidoc      |  9 +++++++--
 qutebrowser/browser/commands.py | 16 ++++++++++++++--
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc
index f54bd6010..dcc90af65 100644
--- a/doc/help/commands.asciidoc
+++ b/doc/help/commands.asciidoc
@@ -146,13 +146,18 @@ Close the current window.
 
 [[download]]
 === download
-Syntax: +:download ['url'] ['dest']+
+Syntax: +:download [*--dest* 'DEST'] ['url'] ['dest-old']+
 
 Download a given URL, or current page if no URL given.
 
+The form `:download [url] [dest]` is deprecated, use `:download --dest [dest] [url]` instead.
+
 ==== positional arguments
 * +'url'+: The URL to download. If not given, download the current page.
-* +'dest'+: The file path to write the download to, or not given to ask.
+* +'dest-old'+: (deprecated) Same as dest.
+
+==== optional arguments
+* +*-d*+, +*--dest*+: The file path to write the download to, or not given to ask.
 
 [[download-cancel]]
 === download-cancel
diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index a2cfff718..633285727 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1140,13 +1140,25 @@ class CommandDispatcher:
                 cur.inspector.show()
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def download(self, url=None, dest=None):
+    def download(self, url=None, dest_old=None, *, dest=None):
         """Download a given URL, or current page if no URL given.
 
+        The form `:download [url] [dest]` is deprecated, use `:download --dest
+        [dest] [url]` instead.
+
         Args:
             url: The URL to download. If not given, download the current page.
+            dest_old: (deprecated) Same as dest.
             dest: The file path to write the download to, or None to ask.
         """
+        if dest_old is not None:
+            message.warning('current', ":download [url] [dest] is deprecated -"
+                            " use download --dest [dest] [url]")
+            if dest is not None:
+                raise cmdexc.CommandError("Can't give two destinations for the"
+                                          " download.")
+            dest = dest_old
+
         download_manager = objreg.get('download-manager', scope='window',
                                       window=self._win_id)
         if url:
@@ -1155,7 +1167,7 @@ class CommandDispatcher:
             download_manager.get(url, filename=dest)
         else:
             page = self._current_widget().page()
-            download_manager.get(self._current_url(), page=page)
+            download_manager.get(self._current_url(), page=page, filename=dest)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def download_whole(self, dest=None):

From d1f8d29c20ee2a5a113e850f5195d558235dd024 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 23:22:21 +0200
Subject: [PATCH 47/66] Add --mhtml flag to :download

And remove :download-whole command.
---
 doc/help/commands.asciidoc      | 13 ++-----------
 qutebrowser/browser/commands.py | 16 +++++++++++-----
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc
index dcc90af65..22219a994 100644
--- a/doc/help/commands.asciidoc
+++ b/doc/help/commands.asciidoc
@@ -19,7 +19,6 @@
 |<<download-open,download-open>>|Open the last/[count]th download.
 |<<download-remove,download-remove>>|Remove the last/[count]th download from the list.
 |<<download-retry,download-retry>>|Retry the first failed/[count]th download.
-|<<download-whole,download-whole>>|Download the current page as a MHTML file, including all assets.
 |<<forward,forward>>|Go forward in the history of the current tab.
 |<<fullscreen,fullscreen>>|Toggle fullscreen mode.
 |<<help,help>>|Show help about a command or setting.
@@ -146,7 +145,7 @@ Close the current window.
 
 [[download]]
 === download
-Syntax: +:download [*--dest* 'DEST'] ['url'] ['dest-old']+
+Syntax: +:download [*--mhtml*] [*--dest* 'DEST'] ['url'] ['dest-old']+
 
 Download a given URL, or current page if no URL given.
 
@@ -157,6 +156,7 @@ The form `:download [url] [dest]` is deprecated, use `:download --dest [dest] [u
 * +'dest-old'+: (deprecated) Same as dest.
 
 ==== optional arguments
+* +*-m*+, +*--mhtml*+: Download the current page and all assets as mhtml file.
 * +*-d*+, +*--dest*+: The file path to write the download to, or not given to ask.
 
 [[download-cancel]]
@@ -203,15 +203,6 @@ Retry the first failed/[count]th download.
 ==== count
 The index of the download to cancel.
 
-[[download-whole]]
-=== download-whole
-Syntax: +:download-whole ['dest']+
-
-Download the current page as a MHTML file, including all assets.
-
-==== positional arguments
-* +'dest'+: The file path to write the download to.
-
 [[forward]]
 === forward
 Syntax: +:forward [*--tab*] [*--bg*] [*--window*]+
diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 633285727..aa9dd2e4d 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1140,7 +1140,7 @@ class CommandDispatcher:
                 cur.inspector.show()
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def download(self, url=None, dest_old=None, *, dest=None):
+    def download(self, url=None, dest_old=None, *, mhtml=False, dest=None):
         """Download a given URL, or current page if no URL given.
 
         The form `:download [url] [dest]` is deprecated, use `:download --dest
@@ -1150,6 +1150,7 @@ class CommandDispatcher:
             url: The URL to download. If not given, download the current page.
             dest_old: (deprecated) Same as dest.
             dest: The file path to write the download to, or None to ask.
+            mhtml: Download the current page and all assets as mhtml file.
         """
         if dest_old is not None:
             message.warning('current', ":download [url] [dest] is deprecated -"
@@ -1162,15 +1163,20 @@ class CommandDispatcher:
         download_manager = objreg.get('download-manager', scope='window',
                                       window=self._win_id)
         if url:
+            if mhtml:
+                raise cmdexc.CommandError("Can only download the current page"
+                                          " as mhtml.")
             url = urlutils.qurl_from_user_input(url)
             urlutils.raise_cmdexc_if_invalid(url)
             download_manager.get(url, filename=dest)
         else:
-            page = self._current_widget().page()
-            download_manager.get(self._current_url(), page=page, filename=dest)
+            if mhtml:
+                self._download_mhtml(dest)
+            else:
+                page = self._current_widget().page()
+                download_manager.get(self._current_url(), page=page, filename=dest)
 
-    @cmdutils.register(instance='command-dispatcher', scope='window')
-    def download_whole(self, dest=None):
+    def _download_mhtml(self, dest=None):
         """Download the current page as a MHTML file, including all assets.
 
         Args:

From 12a9deb9bc69eaff72df38bb3d2ac6165cdec849 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Wed, 7 Oct 2015 23:53:03 +0200
Subject: [PATCH 48/66] Fix lints

---
 qutebrowser/browser/commands.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index aa9dd2e4d..f3bb32513 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1140,7 +1140,7 @@ class CommandDispatcher:
                 cur.inspector.show()
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def download(self, url=None, dest_old=None, *, mhtml=False, dest=None):
+    def download(self, url=None, dest_old=None, *, mhtml_=False, dest=None):
         """Download a given URL, or current page if no URL given.
 
         The form `:download [url] [dest]` is deprecated, use `:download --dest
@@ -1150,7 +1150,7 @@ class CommandDispatcher:
             url: The URL to download. If not given, download the current page.
             dest_old: (deprecated) Same as dest.
             dest: The file path to write the download to, or None to ask.
-            mhtml: Download the current page and all assets as mhtml file.
+            mhtml_: Download the current page and all assets as mhtml file.
         """
         if dest_old is not None:
             message.warning('current', ":download [url] [dest] is deprecated -"
@@ -1163,18 +1163,19 @@ class CommandDispatcher:
         download_manager = objreg.get('download-manager', scope='window',
                                       window=self._win_id)
         if url:
-            if mhtml:
+            if mhtml_:
                 raise cmdexc.CommandError("Can only download the current page"
                                           " as mhtml.")
             url = urlutils.qurl_from_user_input(url)
             urlutils.raise_cmdexc_if_invalid(url)
             download_manager.get(url, filename=dest)
         else:
-            if mhtml:
+            if mhtml_:
                 self._download_mhtml(dest)
             else:
                 page = self._current_widget().page()
-                download_manager.get(self._current_url(), page=page, filename=dest)
+                download_manager.get(self._current_url(), page=page,
+                                     filename=dest)
 
     def _download_mhtml(self, dest=None):
         """Download the current page as a MHTML file, including all assets.

From 3a2bb2d348f5517a3e4eda52b03d563023f72d78 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Thu, 8 Oct 2015 18:04:53 +0200
Subject: [PATCH 49/66] Add cssutils to README and utils/version.py

---
 README.asciidoc                  |  3 +++
 qutebrowser/utils/version.py     |  1 +
 tests/unit/utils/test_version.py | 14 ++++++++++----
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/README.asciidoc b/README.asciidoc
index 43240d97c..04ca13a35 100644
--- a/README.asciidoc
+++ b/README.asciidoc
@@ -100,6 +100,9 @@ The following software and libraries are required to run qutebrowser:
 * http://pygments.org/[pygments]
 * http://pyyaml.org/wiki/PyYAML[PyYAML]
 
+The following libraries are optional and provide a better user experience:
+* http://cthedot.de/cssutils/[cssutils]
+
 To generate the documentation for the `:help` command, when using the git
 repository (rather than a release), http://asciidoc.org/[asciidoc] is needed.
 
diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py
index 6975ba5fa..91366bd28 100644
--- a/qutebrowser/utils/version.py
+++ b/qutebrowser/utils/version.py
@@ -133,6 +133,7 @@ def _module_versions():
         ('jinja2', ['__version__']),
         ('pygments', ['__version__']),
         ('yaml', ['__version__']),
+        ('cssutils', ['__version__']),
     ])
     for name, attributes in modules.items():
         try:
diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py
index 8502c6878..d759b44dd 100644
--- a/tests/unit/utils/test_version.py
+++ b/tests/unit/utils/test_version.py
@@ -324,6 +324,7 @@ class ImportFake:
             'jinja2': True,
             'pygments': True,
             'yaml': True,
+            'cssutils': True,
         }
         self.version_attribute = '__version__'
         self.version = '1.2.3'
@@ -383,12 +384,13 @@ class TestModuleVersions:
         """Test with all modules present in version 1.2.3."""
         expected = ['sip: yes', 'colorlog: yes', 'colorama: 1.2.3',
                     'pypeg2: 1.2.3', 'jinja2: 1.2.3', 'pygments: 1.2.3',
-                    'yaml: 1.2.3']
+                    'yaml: 1.2.3', 'cssutils: 1.2.3']
         assert version._module_versions() == expected
 
     @pytest.mark.parametrize('module, idx, expected', [
         ('colorlog', 1, 'colorlog: no'),
         ('colorama', 2, 'colorama: no'),
+        ('cssutils', 7, 'cssutils: no'),
     ])
     def test_missing_module(self, module, idx, expected, import_fake):
         """Test with a module missing.
@@ -404,12 +406,13 @@ class TestModuleVersions:
     @pytest.mark.parametrize('value, expected', [
         ('VERSION', ['sip: yes', 'colorlog: yes', 'colorama: 1.2.3',
                      'pypeg2: yes', 'jinja2: yes', 'pygments: yes',
-                     'yaml: yes']),
+                     'yaml: yes', 'cssutils: yes']),
         ('SIP_VERSION_STR', ['sip: 1.2.3', 'colorlog: yes', 'colorama: yes',
                              'pypeg2: yes', 'jinja2: yes', 'pygments: yes',
-                             'yaml: yes']),
+                             'yaml: yes', 'cssutils: yes']),
         (None, ['sip: yes', 'colorlog: yes', 'colorama: yes', 'pypeg2: yes',
-                'jinja2: yes', 'pygments: yes', 'yaml: yes']),
+                'jinja2: yes', 'pygments: yes', 'yaml: yes',
+                'cssutils: yes']),
     ])
     def test_version_attribute(self, value, expected, import_fake):
         """Test with a different version attribute.
@@ -432,6 +435,9 @@ class TestModuleVersions:
         ('jinja2', True),
         ('pygments', True),
         ('yaml', True),
+        pytest.mark.xfail(sys.version_info >= (3, 5), reason='cssutils 1.0 and'
+                          ' earlier are broken on Python 3.5')(('cssutils',
+                                                                True)),
     ])
     def test_existing_attributes(self, name, has_version):
         """Check if all dependencies have an expected __version__ attribute.

From ed8a6a4c7bd2bd367027d6e54067469e622b6116 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 11 Oct 2015 11:06:00 +0200
Subject: [PATCH 50/66] Update to cssutils 1.0.1

This fixes cssutils on Python 3.5 (yay!).
---
 requirements.txt                 | 2 +-
 tests/unit/browser/test_mhtml.py | 4 +---
 tests/unit/utils/test_version.py | 4 +---
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 9e07256bd..d5dd44c08 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,4 @@ pyPEG2==2.15.2
 PyYAML==3.11
 colorama==0.3.3
 colorlog==2.6.0
-cssutils==1.0
+cssutils==1.0.1
diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 3ac115ca5..287bb1bc1 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -230,9 +230,7 @@ def test_empty_content_type(checker):
 
 
 @pytest.mark.parametrize('has_cssutils', [
-    # TODO: remove xfail once new cssutils is released
-    pytest.mark.xfail(sys.version_info >= (3, 5), reason='https://'
-                      'bitbucket.org/cthedot/cssutils/issues/52')(True),
+    True,
     False,
 ])
 @pytest.mark.parametrize('inline, style, expected_urls', [
diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py
index d759b44dd..eb398156f 100644
--- a/tests/unit/utils/test_version.py
+++ b/tests/unit/utils/test_version.py
@@ -435,9 +435,7 @@ class TestModuleVersions:
         ('jinja2', True),
         ('pygments', True),
         ('yaml', True),
-        pytest.mark.xfail(sys.version_info >= (3, 5), reason='cssutils 1.0 and'
-                          ' earlier are broken on Python 3.5')(('cssutils',
-                                                                True)),
+        ('cssutils', True),
     ])
     def test_existing_attributes(self, name, has_version):
         """Check if all dependencies have an expected __version__ attribute.

From f2f9529af739d047801c66052abb8b2c4fedbc67 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sun, 11 Oct 2015 11:51:22 +0200
Subject: [PATCH 51/66] Remove sys import in test_mhtml

---
 tests/unit/browser/test_mhtml.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index 287bb1bc1..d19d6e1c6 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -2,8 +2,6 @@
 import io
 import textwrap
 import re
-import sys
-
 import pytest
 
 from qutebrowser.browser import mhtml

From a1e0ccb78725abb94cfc1d1ad489ea6fc136e12a Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 19 Oct 2015 11:55:11 +0200
Subject: [PATCH 52/66] Fix spelling/style.

---
 qutebrowser/browser/mhtml.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 56fc600e6..1ac39be94 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -121,11 +121,12 @@ def _get_css_imports(data, inline=False):
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
+# Encode the file using base64 encoding.
 E_BASE64 = email.encoders.encode_base64
-"""Encode the file using base64 encoding"""
 
+
+# Encode the file using MIME quoted-printable encoding.
 E_QUOPRI = email.encoders.encode_quopri
-"""Encode the file using MIME quoted-printable encoding."""
 
 
 class MHTMLWriter():
@@ -136,14 +137,13 @@ class MHTMLWriter():
         root_content: The root content as bytes.
         content_location: The url of the page as str.
         content_type: The MIME-type of the root content as str.
-        _files: Mapping of location->_File struct.
+        _files: Mapping of location->_File namedtuple.
     """
 
     def __init__(self, root_content, content_location, content_type):
         self.root_content = root_content
         self.content_location = content_location
         self.content_type = content_type
-
         self._files = {}
 
     def add_file(self, location, content, content_type=None,
@@ -165,7 +165,7 @@ class MHTMLWriter():
         """Output the MHTML file to the given file-like object.
 
         Args:
-            fp: The file-object, openend in "wb" mode.
+            fp: The file-object, opened in "wb" mode.
         """
         msg = email.mime.multipart.MIMEMultipart(
             'related', '---=_qute-{}'.format(uuid.uuid4()))

From 8bb887ddabe5ae0cbbd6523600217536987d3bcc Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 19 Oct 2015 12:13:49 +0200
Subject: [PATCH 53/66] Specify window and tab instead of 'current'

---
 qutebrowser/browser/commands.py | 12 ++++++++----
 qutebrowser/browser/mhtml.py    | 17 ++++++++++-------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index f3bb32513..7588fa424 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1153,8 +1153,8 @@ class CommandDispatcher:
             mhtml_: Download the current page and all assets as mhtml file.
         """
         if dest_old is not None:
-            message.warning('current', ":download [url] [dest] is deprecated -"
-                            " use download --dest [dest] [url]")
+            message.warning(self._win_id, ":download [url] [dest] is deprecated"
+                            " - use download --dest [dest] [url]")
             if dest is not None:
                 raise cmdexc.CommandError("Can't give two destinations for the"
                                           " download.")
@@ -1183,6 +1183,7 @@ class CommandDispatcher:
         Args:
             dest: The file path to write the download to.
         """
+        tab_id = self._current_index()
         if dest is None:
             suggested_fn = self._current_title() + ".mht"
             q = usertypes.Question()
@@ -1190,12 +1191,15 @@ class CommandDispatcher:
             q.mode = usertypes.PromptMode.text
             q.completed.connect(q.deleteLater)
             q.default = downloads.path_suggestion(suggested_fn)
-            q.answered.connect(mhtml.start_download_checked)
+            q.answered.connect(functools.partial(
+                mhtml.start_download_checked, win_id=self._win_id,
+                tab_id=tab_id))
             message_bridge = objreg.get("message-bridge", scope="window",
                                         window=self._win_id)
             message_bridge.ask(q, blocking=False)
         else:
-            mhtml.start_download_checked(dest)
+            mhtml.start_download_checked(dest, win_id=self._win_id,
+                                         tab_id=tab_id)
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        deprecated="Use :download instead.")
diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 1ac39be94..e7bdca516 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -415,39 +415,42 @@ class _NoCloseBytesIO(io.BytesIO):  # pylint: disable=no-init
         super().close()
 
 
-def start_download(dest):
+def _start_download(dest, win_id, tab_id):
     """Start downloading the current page and all assets to a MHTML file.
 
     This will overwrite dest if it already exists.
 
     Args:
         dest: The filename where the resulting file should be saved.
+        win_id, tab_id: Specify the tab whose page should be loaded.
     """
+
     dest = os.path.expanduser(dest)
-    web_view = objreg.get('webview', scope='tab', tab='current')
+    web_view = objreg.get('webview', scope='tab', window=win_id, tab=tab_id)
     loader = _Downloader(web_view, dest)
     loader.run()
 
-
-def start_download_checked(dest):
+def start_download_checked(dest, win_id, tab_id):
     """First check if dest is already a file, then start the download.
 
     Args:
         dest: The filename where the resulting file should be saved.
+        win_id, tab_id: Specify the tab whose page should be loaded.
     """
     # start_download will call os.path.expanduser on dest too, so no need to
     # overwrite dest. We just want to make sure that we're checking
     # the right path here. This also means that the user question will show the
     # original path, not the expanded.
     if not os.path.isfile(os.path.expanduser(dest)):
-        start_download(dest)
+        _start_download(dest, win_id=win_id, tab_id=tab_id)
         return
 
     q = usertypes.Question()
     q.mode = usertypes.PromptMode.yesno
     q.text = "{} exists. Overwrite?".format(dest)
     q.completed.connect(q.deleteLater)
-    q.answered_yes.connect(functools.partial(start_download, dest))
+    q.answered_yes.connect(functools.partial(
+        _start_download, dest, win_id=win_id, tab_id=tab_id))
     message_bridge = objreg.get('message-bridge', scope='window',
-                                window='current')
+                                window=win_id)
     message_bridge.ask(q, blocking=False)

From ae8a9b87987d989e0252f1249749efcaee129024 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 19 Oct 2015 14:05:59 +0200
Subject: [PATCH 54/66] Handle non-ASCII in headers/url better

---
 qutebrowser/browser/mhtml.py      | 14 +++++++++-----
 qutebrowser/utils/urlutils.py     |  9 +++++++++
 tests/unit/utils/test_urlutils.py | 13 +++++++++++++
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index e7bdca516..82bb20ed0 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -33,7 +33,7 @@ import email.mime.multipart
 from PyQt5.QtCore import QUrl
 
 from qutebrowser.browser import webelem
-from qutebrowser.utils import log, objreg, message, usertypes
+from qutebrowser.utils import log, objreg, message, usertypes, utils, urlutils
 
 try:
     import cssutils
@@ -237,7 +237,7 @@ class _Downloader():
 
         self.writer = MHTMLWriter(
             web_frame.toHtml().encode('utf-8'),
-            content_location=web_url.toString(),
+            content_location=urlutils.encoded_url(web_url),
             # I've found no way of getting the content type of a QWebView, but
             # since we're using .toHtml, it's probably safe to say that the
             # content-type is HTML
@@ -347,8 +347,12 @@ class _Downloader():
                 self.fetch_url(absolute_url)
 
         encode = E_QUOPRI if mime.startswith('text/') else E_BASE64
-        self.writer.add_file(url.toString(), item.fileobj.getvalue(), mime,
-                             encode)
+        # Our MHTML handler refuses non-ASCII headers. This will replace every
+        # non-ASCII char with '?'. This is probably okay, as official Content-
+        # Type headers contain ASCII only anyway. Anything else is madness.
+        mime = utils.force_encoding(mime, 'ascii')
+        self.writer.add_file(urlutils.encoded_url(url),
+                             item.fileobj.getvalue(), mime, encode)
         item.fileobj.actual_close()
         if self.pending_downloads:
             return
@@ -369,7 +373,7 @@ class _Downloader():
             log.downloads.debug("Oops! Download already gone: %s", item)
             return
         item.fileobj.actual_close()
-        self.writer.add_file(url.toString(), b'')
+        self.writer.add_file(ulrutils.encoded_url(url), b'')
         if self.pending_downloads:
             return
         self.finish_file()
diff --git a/qutebrowser/utils/urlutils.py b/qutebrowser/utils/urlutils.py
index e372dc65f..01c366c25 100644
--- a/qutebrowser/utils/urlutils.py
+++ b/qutebrowser/utils/urlutils.py
@@ -438,6 +438,15 @@ def same_domain(url1, url2):
     return domain1 == domain2
 
 
+def encoded_url(url):
+    """Return the fully encoded url as string.
+
+    Args:
+        url: The url to encode as QUrl.
+    """
+    return bytes(url.toEncoded()).decode('ascii')
+
+
 class IncDecError(Exception):
 
     """Exception raised by incdec_number on problems.
diff --git a/tests/unit/utils/test_urlutils.py b/tests/unit/utils/test_urlutils.py
index 4a19df689..5496a019b 100644
--- a/tests/unit/utils/test_urlutils.py
+++ b/tests/unit/utils/test_urlutils.py
@@ -527,6 +527,19 @@ def test_same_domain_invalid_url(url1, url2):
     with pytest.raises(urlutils.InvalidUrlError):
         urlutils.same_domain(QUrl(url1), QUrl(url2))
 
+
+@pytest.mark.parametrize('url, expected', [
+    ('http://example.com', 'http://example.com'),
+    ('http://ünicode.com', 'http://xn--nicode-2ya.com'),
+    ('http://foo.bar/?header=text/pläin',
+     'http://foo.bar/?header=text/pl%C3%A4in'),
+])
+def test_encoded_url(url, expected):
+    """Test encoded_url"""
+    url = QUrl(url)
+    assert urlutils.encoded_url(url) == expected
+
+
 class TestIncDecNumber:
 
     """Tests for urlutils.incdec_number()."""

From a780325a3ae279785a1f6b9c020c2b5cde925c46 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 19 Oct 2015 20:08:37 +0200
Subject: [PATCH 55/66] Allow directories to be entered as destination

The filename will then default to 'page title.mht'
---
 qutebrowser/browser/downloads.py | 63 +++++++++++++++++---------------
 qutebrowser/browser/mhtml.py     | 41 +++++++++++++++------
 2 files changed, 63 insertions(+), 41 deletions(-)

diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index 3478b0a87..efe7f21e9 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -49,7 +49,7 @@ ModelRole = usertypes.enum('ModelRole', ['item'], start=Qt.UserRole,
 RetryInfo = collections.namedtuple('RetryInfo', ['request', 'manager'])
 
 # Remember the last used directory
-_last_used_directory = None
+last_used_directory = None
 
 
 # All REFRESH_INTERVAL milliseconds, speeds will be recalculated and downloads
@@ -57,13 +57,13 @@ _last_used_directory = None
 REFRESH_INTERVAL = 500
 
 
-def _download_dir():
+def download_dir():
     """Get the download directory to use."""
     directory = config.get('storage', 'download-directory')
     remember_dir = config.get('storage', 'remember-download-directory')
 
-    if remember_dir and _last_used_directory is not None:
-        return _last_used_directory
+    if remember_dir and last_used_directory is not None:
+        return last_used_directory
     elif directory is None:
         return standarddir.download()
     else:
@@ -79,15 +79,36 @@ def path_suggestion(filename):
     suggestion = config.get('completion', 'download-path-suggestion')
     if suggestion == 'path':
         # add trailing '/' if not present
-        return os.path.join(_download_dir(), '')
+        return os.path.join(download_dir(), '')
     elif suggestion == 'filename':
         return filename
     elif suggestion == 'both':
-        return os.path.join(_download_dir(), filename)
+        return os.path.join(download_dir(), filename)
     else:
         raise ValueError("Invalid suggestion value {}!".format(suggestion))
 
 
+def create_full_filename(basename, filename):
+    """Create a full filename based on the given basename and filename.
+
+    Args:
+        basename: The basename to use if filename is a directory.
+        filename: The path to a folder or file where you want to save.
+
+    Return:
+        The full absolute path, or None if filename creation was not possible.
+    """
+    if os.path.isabs(filename) and os.path.isdir(filename):
+        # We got an absolute directory from the user, so we save it under
+        # the default filename in that directory.
+        return os.path.join(filename, basename)
+    elif os.path.isabs(filename):
+        # We got an absolute filename from the user, so we save it under
+        # that filename.
+        return filename
+    return None
+
+
 class DownloadItemStats(QObject):
 
     """Statistics (bytes done, total bytes, time, etc.) about a download.
@@ -447,7 +468,7 @@ class DownloadItem(QObject):
             filename: The full filename to save the download to.
                       None: special value to stop the download.
         """
-        global _last_used_directory
+        global last_used_directory
         if self.fileobj is not None:
             raise ValueError("fileobj was already set! filename: {}, "
                              "existing: {}, fileobj {}".format(
@@ -457,13 +478,16 @@ class DownloadItem(QObject):
         # See https://github.com/The-Compiler/qutebrowser/issues/427
         encoding = sys.getfilesystemencoding()
         filename = utils.force_encoding(filename, encoding)
-        if not self._create_full_filename(filename):
+        self._filename = create_full_filename(self.basename, filename)
+        if self._filename is None:
             # We only got a filename (without directory) or a relative path
             # from the user, so we append that to the default directory and
             # try again.
-            self._create_full_filename(os.path.join(_download_dir(), filename))
+            self._filename = create_full_filename(
+                self.basename, os.path.join(download_dir(), filename))
 
-        _last_used_directory = os.path.dirname(self._filename)
+        self.basename = os.path.basename(self._filename)
+        last_used_directory = os.path.dirname(self._filename)
 
         log.downloads.debug("Setting filename to {}".format(filename))
         if os.path.isfile(self._filename):
@@ -480,25 +504,6 @@ class DownloadItem(QObject):
         else:
             self._create_fileobj()
 
-    def _create_full_filename(self, filename):
-        """Try to create the full filename.
-
-        Return:
-            True if the full filename was created, False otherwise.
-        """
-        if os.path.isabs(filename) and os.path.isdir(filename):
-            # We got an absolute directory from the user, so we save it under
-            # the default filename in that directory.
-            self._filename = os.path.join(filename, self.basename)
-            return True
-        elif os.path.isabs(filename):
-            # We got an absolute filename from the user, so we save it under
-            # that filename.
-            self._filename = filename
-            self.basename = os.path.basename(self._filename)
-            return True
-        return False
-
     def set_fileobj(self, fileobj):
         """"Set the file object to write the download to.
 
diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 82bb20ed0..23170f621 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -23,6 +23,7 @@ import functools
 import io
 import os
 import re
+import sys
 import collections
 import uuid
 import email.policy
@@ -32,7 +33,7 @@ import email.mime.multipart
 
 from PyQt5.QtCore import QUrl
 
-from qutebrowser.browser import webelem
+from qutebrowser.browser import webelem, downloads
 from qutebrowser.utils import log, objreg, message, usertypes, utils, urlutils
 
 try:
@@ -373,7 +374,7 @@ class _Downloader():
             log.downloads.debug("Oops! Download already gone: %s", item)
             return
         item.fileobj.actual_close()
-        self.writer.add_file(ulrutils.encoded_url(url), b'')
+        self.writer.add_file(urlutils.encoded_url(url), b'')
         if self.pending_downloads:
             return
         self.finish_file()
@@ -428,12 +429,11 @@ def _start_download(dest, win_id, tab_id):
         dest: The filename where the resulting file should be saved.
         win_id, tab_id: Specify the tab whose page should be loaded.
     """
-
-    dest = os.path.expanduser(dest)
     web_view = objreg.get('webview', scope='tab', window=win_id, tab=tab_id)
     loader = _Downloader(web_view, dest)
     loader.run()
 
+
 def start_download_checked(dest, win_id, tab_id):
     """First check if dest is already a file, then start the download.
 
@@ -441,20 +441,37 @@ def start_download_checked(dest, win_id, tab_id):
         dest: The filename where the resulting file should be saved.
         win_id, tab_id: Specify the tab whose page should be loaded.
     """
-    # start_download will call os.path.expanduser on dest too, so no need to
-    # overwrite dest. We just want to make sure that we're checking
-    # the right path here. This also means that the user question will show the
-    # original path, not the expanded.
-    if not os.path.isfile(os.path.expanduser(dest)):
-        _start_download(dest, win_id=win_id, tab_id=tab_id)
+    # The default name is 'page title.mht'
+    title = (objreg.get('webview', scope='tab', window=win_id, tab=tab_id)
+             .title())
+    default_name = title + '.mht'
+
+    # Remove characters which cannot be expressed in the file system encoding
+    encoding = sys.getfilesystemencoding()
+    default_name = utils.force_encoding(default_name, encoding)
+    dest = utils.force_encoding(dest, encoding)
+
+    dest = os.path.expanduser(dest)
+
+    # See if we already have an absolute path
+    path = downloads.create_full_filename(default_name, dest)
+    if path is None:
+        # We still only have a relative path, prepend download_dir and
+        # try again.
+        path = downloads.create_full_filename(
+            default_name, os.path.join(downloads.download_dir(), dest))
+    downloads.last_used_directory = os.path.dirname(path)
+
+    if not os.path.isfile(path):
+        _start_download(path, win_id=win_id, tab_id=tab_id)
         return
 
     q = usertypes.Question()
     q.mode = usertypes.PromptMode.yesno
-    q.text = "{} exists. Overwrite?".format(dest)
+    q.text = "{} exists. Overwrite?".format(path)
     q.completed.connect(q.deleteLater)
     q.answered_yes.connect(functools.partial(
-        _start_download, dest, win_id=win_id, tab_id=tab_id))
+        _start_download, path, win_id=win_id, tab_id=tab_id))
     message_bridge = objreg.get('message-bridge', scope='window',
                                 window=win_id)
     message_bridge.ask(q, blocking=False)

From f34161423c9e2cece6712fa512ce33aa0c386f8e Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 19 Oct 2015 20:17:22 +0200
Subject: [PATCH 56/66] Fix "line too long"

---
 qutebrowser/browser/commands.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 7588fa424..763c2c319 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1153,8 +1153,9 @@ class CommandDispatcher:
             mhtml_: Download the current page and all assets as mhtml file.
         """
         if dest_old is not None:
-            message.warning(self._win_id, ":download [url] [dest] is deprecated"
-                            " - use download --dest [dest] [url]")
+            message.warning(
+                self._win_id, ":download [url] [dest] is deprecated - use"
+                              " download --dest [dest] [url]")
             if dest is not None:
                 raise cmdexc.CommandError("Can't give two destinations for the"
                                           " download.")

From c2218f51cd7fc9e6534d5d4cdb270e4a87d1a3f0 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sat, 24 Oct 2015 11:21:00 +0200
Subject: [PATCH 57/66] Add mhtml.last_used_directory to vulture whitelist

---
 scripts/dev/run_vulture.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py
index 4d9805332..6715cd201 100644
--- a/scripts/dev/run_vulture.py
+++ b/scripts/dev/run_vulture.py
@@ -80,6 +80,7 @@ def whitelist_generator():
     # https://bitbucket.org/jendrikseipp/vulture/issues/10/
     yield 'qutebrowser.misc.utilcmds.pyeval_output'
     yield 'utils.use_color'
+    yield 'qutebrowser.browser.mhtml.last_used_directory'
 
     # Other false-positives
     yield ('qutebrowser.completion.models.sortfilter.CompletionFilterModel().'

From 252dc9a8bd747c3d1f5b997dd3fb5896cdf68cbd Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sat, 24 Oct 2015 11:32:25 +0200
Subject: [PATCH 58/66] mhtml: Don't attempt to download hostblocked urls

---
 qutebrowser/browser/mhtml.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 23170f621..d6c5de8a1 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -297,6 +297,15 @@ class _Downloader():
 
         log.downloads.debug("loading asset at %s", url)
 
+        host_blocker = objreg.get('host-blocker')
+        if host_blocker.is_blocked(url):
+            log.downloads.debug("Skipping %s, host-blocked", url)
+            # We still need an empty file in the output, QWebView can be pretty
+            # picky about displaying a file correctly when not all assets are
+            # at least referenced in the mhtml file.
+            self.writer.add_file(urlutils.encoded_url(url), b'')
+            return
+
         download_manager = objreg.get('download-manager', scope='window',
                                       window='current')
         item = download_manager.get(url, fileobj=_NoCloseBytesIO(),
@@ -374,6 +383,7 @@ class _Downloader():
             log.downloads.debug("Oops! Download already gone: %s", item)
             return
         item.fileobj.actual_close()
+        # Add a stub file, see comment in .fetch_url() for more information
         self.writer.add_file(urlutils.encoded_url(url), b'')
         if self.pending_downloads:
             return

From 9694b3b54804c594a7a3a1b437998f10c6f20b61 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sat, 24 Oct 2015 11:33:04 +0200
Subject: [PATCH 59/66] downloads: Remove safety check in on_reply_error

Probably not needed after the last commit.
---
 qutebrowser/browser/downloads.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index efe7f21e9..e2aea5dd3 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -590,9 +590,6 @@ class DownloadItem(QObject):
         if code == QNetworkReply.OperationCanceledError:
             return
         else:
-            if self.reply is None:
-                log.downloads.debug("QNetworkReply disappeared %s", self)
-                return
             self._die(self.reply.errorString())
 
     @pyqtSlot()

From 5601c8e7c11f27f14fdd582a3d5aa8bc4af8ec6c Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sat, 24 Oct 2015 11:54:25 +0200
Subject: [PATCH 60/66] mhtml: Skip test if cssutils is unavailable

Instead of failing the test (assert cssutils is not None)
---
 tests/unit/browser/test_mhtml.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/unit/browser/test_mhtml.py b/tests/unit/browser/test_mhtml.py
index d19d6e1c6..7b5fed7be 100644
--- a/tests/unit/browser/test_mhtml.py
+++ b/tests/unit/browser/test_mhtml.py
@@ -228,9 +228,10 @@ def test_empty_content_type(checker):
 
 
 @pytest.mark.parametrize('has_cssutils', [
-    True,
+    pytest.mark.skipif(mhtml.cssutils is None,
+                       reason="requires cssutils")(True),
     False,
-])
+], ids=['with_cssutils', 'no_cssutils'])
 @pytest.mark.parametrize('inline, style, expected_urls', [
     (False, "@import 'default.css'", ['default.css']),
     (False, '@import "default.css"', ['default.css']),
@@ -244,9 +245,7 @@ def test_empty_content_type(checker):
 ])
 def test_css_url_scanner(monkeypatch, has_cssutils, inline, style,
                          expected_urls):
-    if has_cssutils:
-        assert mhtml.cssutils is not None
-    else:
+    if not has_cssutils:
         monkeypatch.setattr('qutebrowser.browser.mhtml.cssutils', None)
     expected_urls.sort()
     urls = mhtml._get_css_imports(style, inline=inline)

From 71533b3456e52fffc0755e0ec6c202bde80ab3c3 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Sat, 24 Oct 2015 12:12:07 +0200
Subject: [PATCH 61/66] mhtml: Add comments describing the hostblock check

---
 qutebrowser/browser/mhtml.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index d6c5de8a1..bc6f0aa6b 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -297,6 +297,10 @@ class _Downloader():
 
         log.downloads.debug("loading asset at %s", url)
 
+        # Using the download manager to download host-blocked urls might crash
+        # qute, see the comments/discussion on
+        # https://github.com/The-Compiler/qutebrowser/pull/962#discussion_r40256987
+        # and https://github.com/The-Compiler/qutebrowser/issues/1053
         host_blocker = objreg.get('host-blocker')
         if host_blocker.is_blocked(url):
             log.downloads.debug("Skipping %s, host-blocked", url)

From 4f01382c64ef5e92823e0eddb22fbc3880a02574 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Fri, 30 Oct 2015 19:59:58 +0100
Subject: [PATCH 62/66] mhtml: Ignore non-http(s) assets

---
 qutebrowser/browser/mhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index bc6f0aa6b..62bb75a55 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -288,7 +288,7 @@ class _Downloader():
         Args:
             url: The file to download as QUrl.
         """
-        if url.scheme() == 'data':
+        if url.scheme() not in {'http', 'https'}:
             return
         # Prevent loading an asset twice
         if url in self.loaded_urls:

From a1bc020fecf786ceb448ebeec27c79bdd7de4f6b Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Fri, 30 Oct 2015 21:08:37 +0100
Subject: [PATCH 63/66] mhtml: Sanitize default filename suggestion

---
 qutebrowser/browser/commands.py |  1 +
 qutebrowser/browser/mhtml.py    |  2 +-
 qutebrowser/utils/utils.py      | 21 +++++++++++++++++++++
 tests/unit/utils/test_utils.py  | 14 ++++++++++++++
 4 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py
index 763c2c319..4e4527cae 100644
--- a/qutebrowser/browser/commands.py
+++ b/qutebrowser/browser/commands.py
@@ -1187,6 +1187,7 @@ class CommandDispatcher:
         tab_id = self._current_index()
         if dest is None:
             suggested_fn = self._current_title() + ".mht"
+            suggested_fn = utils.sanitize_filename(suggested_fn)
             q = usertypes.Question()
             q.text = "Save page to: "
             q.mode = usertypes.PromptMode.text
diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index 62bb75a55..e7272cd78 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -458,7 +458,7 @@ def start_download_checked(dest, win_id, tab_id):
     # The default name is 'page title.mht'
     title = (objreg.get('webview', scope='tab', window=win_id, tab=tab_id)
              .title())
-    default_name = title + '.mht'
+    default_name = utils.sanitize_filename(title + '.mht')
 
     # Remove characters which cannot be expressed in the file system encoding
     encoding = sys.getfilesystemencoding()
diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py
index 01723c704..d0d2a83db 100644
--- a/qutebrowser/utils/utils.py
+++ b/qutebrowser/utils/utils.py
@@ -611,6 +611,27 @@ def force_encoding(text, encoding):
     return text.encode(encoding, errors='replace').decode(encoding)
 
 
+def sanitize_filename(name, replacement='_'):
+    """Replace invalid filename characters.
+
+    Note: This should be used for the basename, as it also removes the path
+    separator.
+
+    Args:
+        name: The filename.
+        replacement: The replacement character (or None).
+    """
+    if replacement is None:
+        replacement = ''
+    # Bad characters taken from Windows, there are even fewer on Linux
+    # See also
+    # https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words
+    bad_chars = '\\/:*?"<>|'
+    for bad_char in bad_chars:
+        name = name.replace(bad_char, replacement)
+    return name
+
+
 def newest_slice(iterable, count):
     """Get an iterable for the n newest items of the given iterable.
 
diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py
index f0ec48531..6d358bee3 100644
--- a/tests/unit/utils/test_utils.py
+++ b/tests/unit/utils/test_utils.py
@@ -839,6 +839,20 @@ def test_force_encoding(inp, enc, expected):
     assert utils.force_encoding(inp, enc) == expected
 
 
+@pytest.mark.parametrize('inp, expected', [
+    ('normal.txt', 'normal.txt'),
+    ('user/repo issues.mht', 'user_repo issues.mht'),
+    ('<Test\\File> - "*?:|', '_Test_File_ - _____'),
+])
+def test_sanitize_filename(inp, expected):
+    assert utils.sanitize_filename(inp) == expected
+
+
+def test_sanitize_filename_empty_replacement():
+    name = '/<Bad File>/'
+    assert utils.sanitize_filename(name, replacement=None) == 'Bad File'
+
+
 class TestNewestSlice:
 
     """Test newest_slice."""

From f79722975f4016155dc94949e7804a9ffb773b14 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 9 Nov 2015 17:06:50 +0100
Subject: [PATCH 64/66] Fix _download_dir reference

---
 qutebrowser/browser/downloads.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index e2aea5dd3..078fa622f 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -737,7 +737,7 @@ class DownloadManager(QAbstractListModel):
             prompt_download_directory = config.get(
                 'storage', 'prompt-download-directory')
         if not prompt_download_directory and not fileobj:
-            filename = _download_dir()
+            filename = download_dir()
 
         if fileobj is not None or filename is not None:
             return self.fetch_request(request,
@@ -832,7 +832,7 @@ class DownloadManager(QAbstractListModel):
         prompt_download_directory = config.get('storage',
                                                'prompt-download-directory')
         if not prompt_download_directory and not fileobj:
-            filename = _download_dir()
+            filename = download_dir()
 
         if filename is not None:
             download.set_filename(filename)

From c0535727ef4dda65e837b45ac1382b7c64ab5564 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Mon, 9 Nov 2015 22:28:24 +0100
Subject: [PATCH 65/66] Only download elements with rel={stylesheet,icon}

Websites may set the rel attribute to whatever they want, so we just
care about stylesheets and icons and not the other stuff like wss
links (looking at you, GitHub), RSS-feeds, ...
---
 qutebrowser/browser/mhtml.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index e7272cd78..bfdd1ae8f 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -119,6 +119,23 @@ def _get_css_imports(data, inline=False):
         return _get_css_imports_cssutils(data, inline)
 
 
+def _check_rel(element):
+    """Return true if the element's rel attribute fits our criteria.
+
+    rel has to contain 'stylesheet' or 'icon'. Also returns True if the rel
+    attribute is unset.
+
+    Args:
+        element: The WebElementWrapper which should be checked.
+    """
+    if 'rel' not in element:
+        return True
+    must_have = {'stylesheet', 'icon'}
+    rels = [rel.lower() for rel in element['rel'].split(' ')]
+    return any(rel in rels for rel in must_have)
+
+
+
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
@@ -250,6 +267,10 @@ class _Downloader():
 
         for element in elements:
             element = webelem.WebElementWrapper(element)
+            # Websites are free to set whatever rel=... attribute they want.
+            # We just care about stylesheets and icons.
+            if not _check_rel(element):
+                continue
             if 'src' in element:
                 element_url = element['src']
             elif 'href' in element:

From a5ee39c35e82078fff3d0bdada9167ac0fdf1a26 Mon Sep 17 00:00:00 2001
From: Daniel <kingdread@gmx.de>
Date: Tue, 10 Nov 2015 09:45:01 +0100
Subject: [PATCH 66/66] Fix PEP 8

---
 qutebrowser/browser/mhtml.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qutebrowser/browser/mhtml.py b/qutebrowser/browser/mhtml.py
index bfdd1ae8f..7381f2d06 100644
--- a/qutebrowser/browser/mhtml.py
+++ b/qutebrowser/browser/mhtml.py
@@ -135,7 +135,6 @@ def _check_rel(element):
     return any(rel in rels for rel in must_have)
 
 
-
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)