Improve script to print entries grouped by browser

The ua_fetch.py has been re-written entirely to find specific entries for
each browser based on diversity rather than sole popularity. The output
is now formatted to print the entries for each browser grouped together.

qutebrowser/config/configtypes.py

@@ -1564,6 +1564,22 @@ class UserAgent(BaseType):
     def complete(self):
         """Complete a list of common user agents."""
         out = [
+            ('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:41.0) Gecko/20100101 '
+             'Firefox/41.0',
+             "Firefox 41.0  Win7 64-bit"),
+            ('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 '
+             'Firefox/40.0',
+             "Firefox 40.0  Win7 64-bit"),
+
+            ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) '
+             'AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 '
+             'Safari/600.8.9',
+             "Safari 8.0  MacOSX"),
+            ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11) '
+             'AppleWebKit/601.1.56 (KHTML, like Gecko) Version/9.0 '
+             'Safari/601.1.56',
+             "Safari Generic  MacOSX"),
+
             ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, '
              'like Gecko) Chrome/45.0.2454.101 Safari/537.36',
              "Chrome 45.0  Win7 64-bit"),
@@ -1571,29 +1587,12 @@ class UserAgent(BaseType):
              'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 '
              'Safari/537.36',
              "Chrome 45.0  MacOSX"),
-            ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) '
-             'AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 '
-             'Safari/600.8.9',
-             "Safari 8.0  MacOSX"),
             ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, '
              'like Gecko) Chrome/45.0.2454.101 Safari/537.36',
              "Chrome 45.0  Win10 64-bit"),
-            ('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:41.0) Gecko/20100101 '
-             'Firefox/41.0',
-             "Firefox 41.0  Win7 64-bit"),
-            ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11) '
-             'AppleWebKit/601.1.56 (KHTML, like Gecko) Version/9.0 '
-             'Safari/601.1.56',
-             "Safari Generic  MacOSX"),
             ('Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, '
              'like Gecko) Chrome/45.0.2454.101 Safari/537.36',
              "Chrome 45.0  Win8.1 64-bit"),
-            ('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 '
-             'Firefox/40.0',
-             "Firefox 40.0  Win7 64-bit"),
-            ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like '
-             'Gecko',
-             "IE 11.0 for Desktop  Win7 64-bit"),
             ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, '
              'like Gecko) Chrome/45.0.2454.93 Safari/537.36',
              "Chrome 45.0  Win7 64-bit"),
@@ -1605,6 +1604,10 @@ class UserAgent(BaseType):
              'like Gecko) Chrome/45.0.2454.99 Safari/537.36',
              "Chrome 45.0  Win7 64-bit"),
 
+            ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like '
+             'Gecko',
+             "IE 11.0 for Desktop  Win7 64-bit"),
+
             ('Mozilla/5.0 (compatible; Googlebot/2.1; '
              '+http://www.google.com/bot.html',
              "Google Bot"),

scripts/dev/ua_fetch.py

@@ -29,33 +29,81 @@ script is formatted to be pasted into configtypes.py.
 import requests
 from lxml import html   # pylint: disable=import-error
 
-# Fetch list of popular user-agents and store the relevant strings
+
+# Fetch list of popular user-agents and return list of relevant strings
+def fetch():
     url = 'https://techblog.willshouse.com/2012/01/03/most-common-user-agents/'
     page = requests.get(url)
     page = html.fromstring(page.text)
     path = '//*[@id="post-2229"]/div[2]/table/tbody'
-table = page.xpath(path)[0]
+    return page.xpath(path)[0]
-indent = "    "
 
-# Print function defition followed by an automatically fetched list of popular
+
-# user agents and a few additional entries for diversity.
+# Filter the received list based on a look up table. The LUT should be a
-print("%sdef complete(self):" % indent)
+# dictionary of the format {browser: versions}, where 'browser' is the name of
-print("%s\"\"\"Complete a list of common user agents.\"\"\"" % (2 * indent))
+# the browser (eg. "Firefox") as string and 'versions' is a set of different
-print("%sout = [" % (2 * indent))
+# versions of this browser that should be included when found (eg. {"Linux",
-for row in table[:12]:
+# "MacOSX"}). This function returns a dictionary with the same keys as the
-    ua = row[1].text_content()
+# LUT, but storing lists of tuples (user_agent, browser_description) as values.
-    browser = row[2].text_content()
+def filter_list(complete_list, browsers):
-    print("%s(\'%s\',\n%s \"%s\")," % (3 * indent, ua, 3 * indent, browser))
+    table = {}
-print("""
+    for entry in complete_list:
-            ('Mozilla/5.0 (iPhone; CPU iPhone OS 8_1_2 like Mac OS X) '
+        # Tuple of (user_agent, browser_description)
-             'AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 '
+        candidate = (entry[1].text_content(), entry[2].text_content())
-             'Mobile/12B440 Safari/600.1.4',
+        for name in browsers:
-             "Mobile Safari 8.0 iOS"),
+            found = False
-            ('Mozilla/5.0 (Android; Mobile; rv:35.0) Gecko/35.0 Firefox/35.0',
+            if name.lower() in candidate[1].lower():
-             "Firefox 35, Android"),
+                for version in browsers[name]:
-            ('Mozilla/5.0 (Linux; Android 5.0.2; One Build/KTU84L.H4) '
+                    if version.lower() in candidate[1].lower():
-             'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 '
+                        if table.get(name) is None:
-             'Chrome/37.0.0.0 Mobile Safari/537.36',
+                            table[name] = []
-             "Android Browser")
+                        table[name].append(candidate)
-""")
+                        browsers[name].remove(version)
-print("%s]\n%sreturn out\n" % (2 * indent, 2 * indent))
+                        found = True
+                        break
+            if found:
+                break
+    return table
+
+
+# Insert a few additional entries for diversity into the dict (as returned by
+# filter_list())
+def add_diversity(table):
+    table["Obscure"] = [
+        ('Mozilla/5.0 (compatible; Googlebot/2.1; '
+         '+http://www.google.com/bot.html',
+         "Google Bot"),
+        ('Wget/1.16.1 (linux-gnu)',
+         "wget 1.16.1"),
+        ('curl/7.40.0',
+         "curl 7.40.0")
+    ]
+    return table
+
+
+if __name__ == '__main__':
+    fetched = fetch()
+    lut = {
+        "Firefox": {"Win", "MacOSX", "Linux", "Android"},
+        "Chrome": {"Win", "MacOSX", "Linux"},
+        "Safari": {"MacOSX", "iOS"}
+        }
+    filtered = filter_list(fetched, lut)
+    filtered = add_diversity(filtered)
+
+    tab = "    "
+    print("%sdef complete(self):" % tab)
+    print("%s\"\"\"Complete a list of common user agents.\"\"\"" % (2 * tab))
+    print("%sout = [" %(2 * tab))
+
+    for browser in ["Firefox", "Safari", "Chrome", "Obscure"]:
+        for it in filtered[browser]:
+            print("%s(\'%s\',\n%s \"%s\")," % (3 * tab, it[0], 3 * tab, it[1]))
+        print("")
+
+    print("""\
+            ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like '
+             'Gecko',
+             "IE 11.0 for Desktop  Win7 64-bit")""")
+
+    print("%s]\n%sreturn out\n" % (2 * tab, 2 * tab))