From f028c3d50d4bd83ce8468ee1022d4216b41da11c Mon Sep 17 00:00:00 2001 From: mvpossum Date: Sun, 11 May 2014 15:37:22 -0300 Subject: [PATCH 1/2] gzip support --- pirate-get.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pirate-get.py b/pirate-get.py index 24f02e3..749bce4 100755 --- a/pirate-get.py +++ b/pirate-get.py @@ -12,6 +12,8 @@ import random from HTMLParser import HTMLParser import argparse from pprint import pprint +from StringIO import StringIO +import gzip class NoRedirection(urllib2.HTTPErrorProcessor): @@ -84,8 +86,14 @@ def main(): # Catch the Ctrl-C exception and exit cleanly try: for page in xrange(pages): - f = urllib2.urlopen(mirror + '/search/' + args.q.replace(" ", "+") + '/' + str(page) + '/7/0') - res = f.read() + request = urllib2.Request(mirror + '/search/' + args.q.replace(" ", "+") + '/' + str(page) + '/7/0') + request.add_header('Accept-encoding', 'gzip') + response = urllib2.urlopen(request) + if response.info().get('Content-Encoding') == 'gzip': + buf = StringIO(response.read()) + res = gzip.GzipFile(fileobj=buf).read() + else: + res = response.read() found = re.findall(""""(magnet\:\?xt=[^"]*)|([^<]+)""", res) # check for a blocked mirror From a1246db22a615b95edeb1196db6ac8ee17f73fab Mon Sep 17 00:00:00 2001 From: mvpossum Date: Sun, 11 May 2014 15:42:18 -0300 Subject: [PATCH 2/2] gzip support --- pirate-get.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pirate-get.py b/pirate-get.py index 749bce4..84c743a 100755 --- a/pirate-get.py +++ b/pirate-get.py @@ -88,12 +88,11 @@ def main(): for page in xrange(pages): request = urllib2.Request(mirror + '/search/' + args.q.replace(" ", "+") + '/' + str(page) + '/7/0') request.add_header('Accept-encoding', 'gzip') - response = urllib2.urlopen(request) - if response.info().get('Content-Encoding') == 'gzip': - buf = StringIO(response.read()) - res = gzip.GzipFile(fileobj=buf).read() - else: - res = response.read() + f = urllib2.urlopen(request) + if f.info().get('Content-Encoding') == 'gzip': + buf = StringIO(f.read()) + f = gzip.GzipFile(fileobj=buf) + res = f.read() found = re.findall(""""(magnet\:\?xt=[^"]*)|([^<]+)""", res) # check for a blocked mirror