mirror of
https://github.com/vikstrous/pirate-get
synced 2025-04-16 00:58:37 +02:00
reimplement fetching multiple pages
This commit is contained in:
parent
3eb240f316
commit
35ecb3041f
GPG Key ID:
BFBAF4C975F76450
@ -134,14 +134,18 @@ def parse_args(args_in):
|
||||
help='specify a sort option', default='SeedersDsc')
|
||||
parser.add_argument('-R', '--recent',
|
||||
action='store_true',
|
||||
help='torrents uploaded in the last 48hours.'
|
||||
help='torrents uploaded in the last 48hours. '
|
||||
'*ignored in searches*')
|
||||
parser.add_argument('-l', '--list-categories',
|
||||
action='store_true',
|
||||
help='list categories')
|
||||
parser.add_argument('--list-sorts', '--list_sorts',
|
||||
action='store_true',
|
||||
help='list Sortable Types')
|
||||
help='list types by which results can be sorted')
|
||||
parser.add_argument('-p', '--pages',
|
||||
default=1, type=int,
|
||||
help='the number of pages to fetch. '
|
||||
'(only used with --recent)')
|
||||
parser.add_argument('-L', '--local', dest='database',
|
||||
help='a csv file containing the Pirate Bay database '
|
||||
'downloaded from '
|
||||
@ -271,6 +275,7 @@ def connect_mirror(mirror, printer, args):
|
||||
url = pirate.torrent.find_api(mirror, args.timeout)
|
||||
results = pirate.torrent.remote(
|
||||
printer=printer,
|
||||
pages=args.pages,
|
||||
category=pirate.torrent.parse_category(printer, args.category),
|
||||
sort=pirate.torrent.parse_sort(printer, args.sort),
|
||||
mode=args.action,
|
||||
|
||||
@ -93,14 +93,14 @@ def build_magnet(name, info_hash):
|
||||
info_hash, parse.quote(name, ''))
|
||||
|
||||
|
||||
def build_request_path(mode, category, terms):
|
||||
def build_request_path(mode, page, category, terms):
|
||||
if mode == 'search':
|
||||
query = '/q.php?q={}&cat={}'.format(' '.join(terms), category)
|
||||
elif mode == 'top':
|
||||
cat = 'all' if category == 0 else category
|
||||
query = '/precompiled/data_top100_{}.json'.format(cat)
|
||||
elif mode == 'recent':
|
||||
query = '/precompiled/data_top100_recent.json'
|
||||
query = '/precompiled/data_top100_recent_{}.json'.format(page)
|
||||
elif mode == 'browse':
|
||||
if category == 0:
|
||||
raise Exception('You must specify a category')
|
||||
@ -111,25 +111,30 @@ def build_request_path(mode, category, terms):
|
||||
return parse.quote(query, '?=&/')
|
||||
|
||||
|
||||
def remote(printer, category, sort, mode, terms, mirror, timeout):
|
||||
query = build_request_path(mode, category, terms)
|
||||
# Catch the Ctrl-C exception and exit cleanly
|
||||
try:
|
||||
req = request.Request(
|
||||
mirror + query,
|
||||
headers=pirate.data.default_headers)
|
||||
def remote(printer, pages, category, sort, mode, terms, mirror, timeout):
|
||||
results = []
|
||||
for i in range(1, pages + 1):
|
||||
query = build_request_path(mode, i, category, terms)
|
||||
|
||||
# Catch the Ctrl-C exception and exit cleanly
|
||||
try:
|
||||
f = request.urlopen(req, timeout=timeout)
|
||||
except urllib.error.URLError as e:
|
||||
raise e
|
||||
req = request.Request(
|
||||
mirror + query,
|
||||
headers=pirate.data.default_headers)
|
||||
try:
|
||||
f = request.urlopen(req, timeout=timeout)
|
||||
except urllib.error.URLError as e:
|
||||
raise e
|
||||
|
||||
if f.info().get('Content-Encoding') == 'gzip':
|
||||
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
||||
except KeyboardInterrupt:
|
||||
printer.print('\nCancelled.')
|
||||
sys.exit(0)
|
||||
if f.info().get('Content-Encoding') == 'gzip':
|
||||
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
|
||||
except KeyboardInterrupt:
|
||||
printer.print('\nCancelled.')
|
||||
sys.exit(0)
|
||||
|
||||
return sort_results(sort, parse_page(f))
|
||||
results.extend(parse_page(f))
|
||||
|
||||
return sort_results(sort, results)
|
||||
|
||||
|
||||
def find_api(mirror, timeout):
|
||||
|
||||
@ -70,18 +70,18 @@ class TestTorrent(unittest.TestCase):
|
||||
def test_request_path(self):
|
||||
# the args are (mode, category, terms)
|
||||
succeed = [
|
||||
(('recent', 0, []), '/precompiled/data_top100_recent.json'),
|
||||
(('recent', 100, []), '/precompiled/data_top100_recent.json'),
|
||||
(('top', 0, []), '/precompiled/data_top100_all.json'),
|
||||
(('top', 100, []), '/precompiled/data_top100_100.json'),
|
||||
(('search', 100, ['abc']), '/q.php?q=abc&cat=100'),
|
||||
(('search', 100, ['abc', 'def']), '/q.php?q=abc%20def&cat=100'),
|
||||
(('search', 100, ['\u1234']), '/q.php?q=%E1%88%B4&cat=100'),
|
||||
(('browse', 100, []), '/q.php?q=category%3A100'),
|
||||
(('recent', 1, 0, []), '/precompiled/data_top100_recent_1.json'),
|
||||
(('recent', 2, 100, []), '/precompiled/data_top100_recent_2.json'),
|
||||
(('top', 1, 0, []), '/precompiled/data_top100_all.json'),
|
||||
(('top', 1, 100, []), '/precompiled/data_top100_100.json'),
|
||||
(('search', 1, 100, ['abc']), '/q.php?q=abc&cat=100'),
|
||||
(('search', 1, 100, ['abc', 'def']), '/q.php?q=abc%20def&cat=100'),
|
||||
(('search', 1, 100, ['\u1234']), '/q.php?q=%E1%88%B4&cat=100'),
|
||||
(('browse', 1, 100, []), '/q.php?q=category%3A100'),
|
||||
]
|
||||
fail = [
|
||||
(('browse', 0, []), Exception),
|
||||
(('asdf', 100, []), Exception)
|
||||
(('browse', 1, 0, []), Exception),
|
||||
(('asdf', 1, 100, []), Exception)
|
||||
]
|
||||
for inp, out in succeed:
|
||||
path = pirate.torrent.build_request_path(*inp)
|
||||
@ -154,7 +154,7 @@ class TestTorrent(unittest.TestCase):
|
||||
with patch('urllib.request.Request', return_value=req_obj) as req:
|
||||
with patch('urllib.request.urlopen', return_value=res_obj) as res:
|
||||
results = pirate.torrent.remote(
|
||||
MagicMock(Printer), 100, sort, 'top',
|
||||
MagicMock(Printer), 1, 100, sort, 'top',
|
||||
[], 'http://example.com', 9)
|
||||
req.assert_called_once_with(
|
||||
'http://example.com/precompiled/data_top100_100.json',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user