mirror of
https://github.com/vikstrous/pirate-get
synced 2025-01-10 10:04:21 +01:00
handle cookies for tpb http requests
The pirate bay (or a hoster in between) added a mechanism to rate-limit requests that forces us to handle cookies.
This commit is contained in:
parent
2e1ae8dc59
commit
7bbfac52e9
@ -4,12 +4,14 @@ import gzip
|
|||||||
import urllib.parse as parse
|
import urllib.parse as parse
|
||||||
import urllib.request as request
|
import urllib.request as request
|
||||||
import shutil
|
import shutil
|
||||||
from io import BytesIO
|
|
||||||
|
import pirate.data
|
||||||
|
|
||||||
import colorama
|
import colorama
|
||||||
import veryprettytable
|
import veryprettytable
|
||||||
|
|
||||||
import pirate.data
|
from io import BytesIO
|
||||||
|
from http.cookiejar import CookieJar
|
||||||
|
|
||||||
|
|
||||||
class Printer:
|
class Printer:
|
||||||
@ -101,12 +103,17 @@ class Printer:
|
|||||||
self.print(table)
|
self.print(table)
|
||||||
|
|
||||||
def descriptions(self, chosen_links, results, site):
|
def descriptions(self, chosen_links, results, site):
|
||||||
|
jar = CookieJar()
|
||||||
|
opener = request.build_opener(
|
||||||
|
request.HTTPErrorProcessor,
|
||||||
|
request.HTTPCookieProcessor(jar))
|
||||||
|
|
||||||
for link in chosen_links:
|
for link in chosen_links:
|
||||||
path = '/torrent/%s/' % results[link]['id']
|
path = '/torrent/%s/' % results[link]['id']
|
||||||
req = request.Request(site + path,
|
req = request.Request(site + path,
|
||||||
headers=pirate.data.default_headers)
|
headers=pirate.data.default_headers)
|
||||||
req.add_header('Accept-encoding', 'gzip')
|
req.add_header('Accept-encoding', 'gzip')
|
||||||
f = request.urlopen(req, timeout=pirate.data.default_timeout)
|
f = opener.open(req, timeout=pirate.data.default_timeout)
|
||||||
|
|
||||||
if f.info().get('Content-Encoding') == 'gzip':
|
if f.info().get('Content-Encoding') == 'gzip':
|
||||||
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
||||||
@ -125,13 +132,18 @@ class Printer:
|
|||||||
self.print(desc, color='zebra_0')
|
self.print(desc, color='zebra_0')
|
||||||
|
|
||||||
def file_lists(self, chosen_links, results, site):
|
def file_lists(self, chosen_links, results, site):
|
||||||
|
jar = CookieJar()
|
||||||
|
opener = request.build_opener(
|
||||||
|
request.HTTPErrorProcessor,
|
||||||
|
request.HTTPCookieProcessor(jar))
|
||||||
|
|
||||||
for link in chosen_links:
|
for link in chosen_links:
|
||||||
path = '/ajax_details_filelist.php'
|
path = '/ajax_details_filelist.php'
|
||||||
query = '?id=' + results[link]['id']
|
query = '?id=' + results[link]['id']
|
||||||
req = request.Request(site + path + query,
|
req = request.Request(site + path + query,
|
||||||
headers=pirate.data.default_headers)
|
headers=pirate.data.default_headers)
|
||||||
req.add_header('Accept-encoding', 'gzip')
|
req.add_header('Accept-encoding', 'gzip')
|
||||||
f = request.urlopen(req, timeout=pirate.data.default_timeout)
|
f = opener.open(req, timeout=pirate.data.default_timeout)
|
||||||
|
|
||||||
if f.info().get('Content-Encoding') == 'gzip':
|
if f.info().get('Content-Encoding') == 'gzip':
|
||||||
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
||||||
|
@ -7,11 +7,11 @@ import urllib.parse as parse
|
|||||||
import urllib.error
|
import urllib.error
|
||||||
import os.path
|
import os.path
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
import pirate.data
|
import pirate.data
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
from http.cookiejar import CookieJar
|
||||||
|
|
||||||
|
|
||||||
parser_regex = r'"(magnet\:\?xt=[^"]*)|<td align="right">([^<]+)</td>'
|
parser_regex = r'"(magnet\:\?xt=[^"]*)|<td align="right">([^<]+)</td>'
|
||||||
@ -152,13 +152,18 @@ def remote(printer, pages, category, sort, mode, terms, mirror):
|
|||||||
|
|
||||||
# Catch the Ctrl-C exception and exit cleanly
|
# Catch the Ctrl-C exception and exit cleanly
|
||||||
try:
|
try:
|
||||||
|
jar = CookieJar()
|
||||||
|
opener = request.build_opener(
|
||||||
|
request.HTTPErrorProcessor,
|
||||||
|
request.HTTPCookieProcessor(jar))
|
||||||
|
|
||||||
for page in range(pages):
|
for page in range(pages):
|
||||||
path = build_request_path(page, category, sort, mode, terms)
|
path = build_request_path(page, category, sort, mode, terms)
|
||||||
|
|
||||||
req = request.Request(mirror + path,
|
req = request.Request(mirror + path,
|
||||||
headers=pirate.data.default_headers)
|
headers=pirate.data.default_headers)
|
||||||
req.add_header('Accept-encoding', 'gzip')
|
req.add_header('Accept-encoding', 'gzip')
|
||||||
f = request.urlopen(req, timeout=pirate.data.default_timeout)
|
f = opener.open(req, timeout=pirate.data.default_timeout)
|
||||||
if f.info().get('Content-Encoding') == 'gzip':
|
if f.info().get('Content-Encoding') == 'gzip':
|
||||||
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
||||||
res = f.read().decode('utf-8')
|
res = f.read().decode('utf-8')
|
||||||
|
Loading…
Reference in New Issue
Block a user