diff --git a/pirate/pirate.py b/pirate/pirate.py index 247ef65..dae16e2 100755 --- a/pirate/pirate.py +++ b/pirate/pirate.py @@ -134,14 +134,18 @@ def parse_args(args_in): help='specify a sort option', default='SeedersDsc') parser.add_argument('-R', '--recent', action='store_true', - help='torrents uploaded in the last 48hours.' + help='torrents uploaded in the last 48hours. ' '*ignored in searches*') parser.add_argument('-l', '--list-categories', action='store_true', help='list categories') parser.add_argument('--list-sorts', '--list_sorts', action='store_true', - help='list Sortable Types') + help='list types by which results can be sorted') + parser.add_argument('-p', '--pages', + default=1, type=int, + help='the number of pages to fetch. ' + '(only used with --recent)') parser.add_argument('-L', '--local', dest='database', help='a csv file containing the Pirate Bay database ' 'downloaded from ' @@ -271,6 +275,7 @@ def connect_mirror(mirror, printer, args): url = pirate.torrent.find_api(mirror, args.timeout) results = pirate.torrent.remote( printer=printer, + pages=args.pages, category=pirate.torrent.parse_category(printer, args.category), sort=pirate.torrent.parse_sort(printer, args.sort), mode=args.action, diff --git a/pirate/torrent.py b/pirate/torrent.py index aa13780..ec39f77 100644 --- a/pirate/torrent.py +++ b/pirate/torrent.py @@ -93,14 +93,14 @@ def build_magnet(name, info_hash): info_hash, parse.quote(name, '')) -def build_request_path(mode, category, terms): +def build_request_path(mode, page, category, terms): if mode == 'search': query = '/q.php?q={}&cat={}'.format(' '.join(terms), category) elif mode == 'top': cat = 'all' if category == 0 else category query = '/precompiled/data_top100_{}.json'.format(cat) elif mode == 'recent': - query = '/precompiled/data_top100_recent.json' + query = '/precompiled/data_top100_recent_{}.json'.format(page) elif mode == 'browse': if category == 0: raise Exception('You must specify a category') @@ -111,25 +111,30 @@ def build_request_path(mode, category, terms): return parse.quote(query, '?=&/') -def remote(printer, category, sort, mode, terms, mirror, timeout): - query = build_request_path(mode, category, terms) - # Catch the Ctrl-C exception and exit cleanly - try: - req = request.Request( - mirror + query, - headers=pirate.data.default_headers) +def remote(printer, pages, category, sort, mode, terms, mirror, timeout): + results = [] + for i in range(1, pages + 1): + query = build_request_path(mode, i, category, terms) + + # Catch the Ctrl-C exception and exit cleanly try: - f = request.urlopen(req, timeout=timeout) - except urllib.error.URLError as e: - raise e + req = request.Request( + mirror + query, + headers=pirate.data.default_headers) + try: + f = request.urlopen(req, timeout=timeout) + except urllib.error.URLError as e: + raise e - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - except KeyboardInterrupt: - printer.print('\nCancelled.') - sys.exit(0) + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + except KeyboardInterrupt: + printer.print('\nCancelled.') + sys.exit(0) - return sort_results(sort, parse_page(f)) + results.extend(parse_page(f)) + + return sort_results(sort, results) def find_api(mirror, timeout): diff --git a/tests/test_torrent.py b/tests/test_torrent.py index fd6ecce..a146cc9 100755 --- a/tests/test_torrent.py +++ b/tests/test_torrent.py @@ -70,18 +70,18 @@ class TestTorrent(unittest.TestCase): def test_request_path(self): # the args are (mode, category, terms) succeed = [ - (('recent', 0, []), '/precompiled/data_top100_recent.json'), - (('recent', 100, []), '/precompiled/data_top100_recent.json'), - (('top', 0, []), '/precompiled/data_top100_all.json'), - (('top', 100, []), '/precompiled/data_top100_100.json'), - (('search', 100, ['abc']), '/q.php?q=abc&cat=100'), - (('search', 100, ['abc', 'def']), '/q.php?q=abc%20def&cat=100'), - (('search', 100, ['\u1234']), '/q.php?q=%E1%88%B4&cat=100'), - (('browse', 100, []), '/q.php?q=category%3A100'), + (('recent', 1, 0, []), '/precompiled/data_top100_recent_1.json'), + (('recent', 2, 100, []), '/precompiled/data_top100_recent_2.json'), + (('top', 1, 0, []), '/precompiled/data_top100_all.json'), + (('top', 1, 100, []), '/precompiled/data_top100_100.json'), + (('search', 1, 100, ['abc']), '/q.php?q=abc&cat=100'), + (('search', 1, 100, ['abc', 'def']), '/q.php?q=abc%20def&cat=100'), + (('search', 1, 100, ['\u1234']), '/q.php?q=%E1%88%B4&cat=100'), + (('browse', 1, 100, []), '/q.php?q=category%3A100'), ] fail = [ - (('browse', 0, []), Exception), - (('asdf', 100, []), Exception) + (('browse', 1, 0, []), Exception), + (('asdf', 1, 100, []), Exception) ] for inp, out in succeed: path = pirate.torrent.build_request_path(*inp) @@ -154,7 +154,7 @@ class TestTorrent(unittest.TestCase): with patch('urllib.request.Request', return_value=req_obj) as req: with patch('urllib.request.urlopen', return_value=res_obj) as res: results = pirate.torrent.remote( - MagicMock(Printer), 100, sort, 'top', + MagicMock(Printer), 1, 100, sort, 'top', [], 'http://example.com', 9) req.assert_called_once_with( 'http://example.com/precompiled/data_top100_100.json',