\s*
(.+?)(?=
)',
- res, re.DOTALL).group(1)
+ res = json.load(f)
# Replace HTML links with markdown style versions
desc = re.sub(r'
]*>(\s*)([^<]+?)(\s*'
- r')', r'\2[\3](\1)\4', desc)
+ r')', r'\2[\3](\1)\4', res['descr'])
- self.print('Description for "%s":' % torrent_name, color='zebra_1')
+ self.print('Description for "{}":'.format(result['name']),
+ color='zebra_1')
self.print(desc, color='zebra_0')
- def file_lists(self, chosen_links, results, site):
- jar = CookieJar()
- opener = request.build_opener(
- request.HTTPErrorProcessor,
- request.HTTPCookieProcessor(jar))
+ def file_lists(self, chosen_links, results, site, timeout):
+ # the API may returns object instead of list
+ def get(obj):
+ try:
+ return obj[0]
+ except KeyError:
+ return obj['0']
for link in chosen_links:
- path = '/ajax_details_filelist.php'
- query = '?id=' + results[link]['id']
- req = request.Request(site + path + query,
- headers=pirate.data.default_headers)
+ result = results[link]
+ req = request.Request(
+ site + '/f.php?id=' + str(result['id']),
+ headers=pirate.data.default_headers)
req.add_header('Accept-encoding', 'gzip')
- f = opener.open(req, timeout=pirate.data.default_timeout)
+ f = request.urlopen(req, timeout=timeout)
if f.info().get('Content-Encoding') == 'gzip':
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
- # TODO: proper html decoding/parsing
- res = f.read().decode('utf-8').replace(' ', ' ')
- if 'File list not available.' in res:
+ res = json.load(f)
+
+ if len(res) == 1 and 'not found' in get(res[0]['name']):
self.print('File list not available.')
return
- files = re.findall(r'
\s*([^<]+?)\s* | \s*([^<]+?)\s*', res)
- name = re.search(r'dn=([^\&]*)', results[link]['magnet'])
- torrent_name = parse.unquote(name.group(1)).replace('+', ' ')
- self.print('Files in "%s":' % torrent_name, color='zebra_1')
+ self.print('Files in {}:'.format(result['name']), color='zebra_1')
cur_color = 'zebra_0'
- for f in files:
- self.print('{0[0]:>11} {0[1]}'.format(f), color=cur_color)
+ for f in res:
+ name = get(f['name'])
+ size = pirate.torrent.pretty_size(int(get(f['size'])))
+ self.print('{:>11} {}'.format(
+ size, name),
+ color=cur_color)
cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1'
diff --git a/pirate/torrent.py b/pirate/torrent.py
index defccbc..ec39f77 100644
--- a/pirate/torrent.py
+++ b/pirate/torrent.py
@@ -8,13 +8,10 @@ import urllib.error
import os.path
import pirate.data
+import json
-from bs4 import BeautifulSoup
+from datetime import datetime
from io import BytesIO
-from http.cookiejar import CookieJar
-
-
-parser_regex = r'"(magnet\:\?xt=[^"]*)| | ([^<]+) | '
def parse_category(printer, category):
@@ -36,208 +33,184 @@ def parse_sort(printer, sort):
sort = int(sort)
except ValueError:
pass
- if sort in pirate.data.sorts.values():
- return sort
- elif sort in pirate.data.sorts.keys():
- return pirate.data.sorts[sort]
+ for key, val in pirate.data.sorts.items():
+ if sort == key or sort == val[0]:
+ return val[1:]
else:
printer.print('Invalid sort ignored', color='WARN')
- return 99
+ return pirate.data.sorts['Default'][1:]
-# TODO:
-# * warn users when using a sort in a mode that doesn't accept sorts
-# * warn users when using search terms in a mode
-# that doesn't accept search terms
-# * same with page parameter for top and top48h
-# * warn the user if trying to use a minor category with top48h
-def build_request_path(page, category, sort, mode, terms):
- if mode == 'browse':
- if(category == 0):
- category = 100
- return '/browse/{}/{}/{}'.format(category, page, sort)
- elif mode == 'recent':
- # This is not a typo. There is no / between 48h and the category.
- path = '/top/48h'
- # only major categories can be used with this mode
- if(category == 0):
- return path + 'all'
- else:
- return path + str(category)
- elif mode == 'top':
- path = '/top/'
- if(category == 0):
- return path + 'all'
- else:
- return path + str(category)
- elif mode == 'search':
- query = urllib.parse.quote_plus(' '.join(terms))
- return '/search/{}/{}/{}/{}'.format(query, page, sort, category)
- else:
- raise Exception('Unknown mode.')
-
-
-# this returns a list of dictionaries
-def parse_page(html):
- soup = BeautifulSoup(html, 'html.parser')
- tables = soup.find_all('table', id='searchResult')
- no_results = re.search(r'No hits\. Try adding an asterisk in '
- r'you search phrase\.', html)
-
- # check for a blocked mirror
- if not tables and not no_results:
- # Contradiction - we found no results,
- # but the page didn't say there were no results.
- # The page is probably not actually the pirate bay,
- # so let's try another mirror
- raise IOError('Blocked mirror detected.')
-
- if no_results:
- return []
-
- # handle ads disguised as fake result tables
- for table in tables:
- results = parse_table(table)
- if results:
- break
- else:
- raise IOError('Mirror does not contain magnets.')
-
- return results
-
-
-def parse_table(table):
+def parse_page(page):
results = []
+ try:
+ data = json.load(page)
+ except json.decoder.JSONDecodeError:
+ raise IOError('invalid JSON in API reply: blocked mirror?')
- # parse the rows one by one (skipping headings)
- for row in table('tr')[1:]:
- # grab info about the row
- row_link = row.find('a', class_='detLink')
- if row_link is None:
- continue
+ if len(data) == 1 and 'No results' in data[0]['name']:
+ return results
- id_ = row_link['href'].split('/')[2]
- seeds, leechers = [i.text for i in row('td')[-2:]]
- magnet_tag = row.find(lambda tag: tag.name == 'a' and
- tag['href'].startswith('magnet'))
- if magnet_tag is None:
- continue
- magnet = magnet_tag['href']
-
- # parse descriptions separately
- description = row.find('font', class_='detDesc').text
- size = re.findall(r'(?<=Size )[0-9.]+\s[KMGT]*[i ]*B',
- description)[0].split()
- uploaded = re.findall(r'(?<=Uploaded ).+(?=\, Size)',
- description)[0]
-
- results.append({
- 'magnet': magnet,
- 'seeds': seeds,
- 'leechers': leechers,
- 'size': size,
- 'uploaded': uploaded,
- 'id': id_
- })
+ for res in data:
+ res['raw_size'] = int(res['size'])
+ res['size'] = pretty_size(int(res['size']))
+ res['magnet'] = build_magnet(res['name'], res['info_hash'])
+ res['info_hash'] = int(res['info_hash'], 16)
+ res['raw_uploaded'] = int(res['added'])
+ res['uploaded'] = pretty_date(res['added'])
+ res['seeders'] = int(res['seeders'])
+ res['leechers'] = int(res['leechers'])
+ res['category'] = int(res['category'])
+ results.append(res)
return results
-def remote(printer, pages, category, sort, mode, terms, mirror):
- res_l = []
+def sort_results(sort, res):
+ key, reverse = sort
+ return sorted(res, key=lambda x: x[key], reverse=reverse)
- if pages < 1:
- raise ValueError('Please provide an integer greater than 0 '
- 'for the number of pages to fetch.')
- # Catch the Ctrl-C exception and exit cleanly
- try:
- jar = CookieJar()
- opener = request.build_opener(
- request.HTTPErrorProcessor,
- request.HTTPCookieProcessor(jar))
+def pretty_size(size):
+ ranges = [('PiB', 1125899906842624),
+ ('TiB', 1099511627776),
+ ('GiB', 1073741824),
+ ('MiB', 1048576),
+ ('KiB', 1024)]
+ for unit, value in ranges:
+ if size >= value:
+ return '{:.1f} {}'.format(size/value, unit)
+ return str(size) + ' B'
- for page in range(pages):
- path = build_request_path(page, category, sort, mode, terms)
- req = request.Request(mirror + path,
- headers=pirate.data.default_headers)
- req.add_header('Accept-encoding', 'gzip')
+def pretty_date(ts):
+ date = datetime.fromtimestamp(int(ts))
+ return date.strftime('%Y-%m-%d %H:%M')
+
+def build_magnet(name, info_hash):
+ return 'magnet:?xt=urn:btih:{}&dn={}'.format(
+ info_hash, parse.quote(name, ''))
+
+
+def build_request_path(mode, page, category, terms):
+ if mode == 'search':
+ query = '/q.php?q={}&cat={}'.format(' '.join(terms), category)
+ elif mode == 'top':
+ cat = 'all' if category == 0 else category
+ query = '/precompiled/data_top100_{}.json'.format(cat)
+ elif mode == 'recent':
+ query = '/precompiled/data_top100_recent_{}.json'.format(page)
+ elif mode == 'browse':
+ if category == 0:
+ raise Exception('You must specify a category')
+ query = '/q.php?q=category:{}'.format(category)
+ else:
+ raise Exception('Invalid mode', mode)
+
+ return parse.quote(query, '?=&/')
+
+
+def remote(printer, pages, category, sort, mode, terms, mirror, timeout):
+ results = []
+ for i in range(1, pages + 1):
+ query = build_request_path(mode, i, category, terms)
+
+ # Catch the Ctrl-C exception and exit cleanly
+ try:
+ req = request.Request(
+ mirror + query,
+ headers=pirate.data.default_headers)
try:
- f = opener.open(req, timeout=pirate.data.default_timeout)
+ f = request.urlopen(req, timeout=timeout)
except urllib.error.URLError as e:
- res = e.fp.read().decode()
- if e.code == 503 and 'cf-browser-verification' in res:
- raise IOError('Cloudflare protected')
raise e
if f.info().get('Content-Encoding') == 'gzip':
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
- res = f.read().decode('utf-8')
+ except KeyboardInterrupt:
+ printer.print('\nCancelled.')
+ sys.exit(0)
- res_l += parse_page(res)
+ results.extend(parse_page(f))
- except KeyboardInterrupt:
- printer.print('\nCancelled.')
- sys.exit(0)
-
- return res_l
+ return sort_results(sort, results)
-def get_torrent(info_hash):
+def find_api(mirror, timeout):
+ # try common paths
+ for path in ['', '/apip', '/api.php?url=']:
+ req = request.Request(mirror + path + '/q.php?q=test&cat=0',
+ headers=pirate.data.default_headers)
+ try:
+ f = request.urlopen(req, timeout=timeout)
+ if f.info().get_content_type() == 'application/json':
+ return mirror + path
+ except urllib.error.URLError as e:
+ res = e.fp.read().decode()
+ if e.code == 503 and 'cf-browser-verification' in res:
+ raise IOError('Cloudflare protected')
+
+ # extract api path from main.js
+ req = request.Request(mirror + '/static/main.js',
+ headers=pirate.data.default_headers)
+ try:
+ f = request.urlopen(req, timeout=timeout)
+ if f.info().get_content_type() == 'application/javascript':
+ match = re.search("var server='([^']+)'", f.read().decode())
+ return mirror + match.group(1)
+ except urllib.error.URLError:
+ raise IOError('API not found: no main.js')
+
+ raise IOError('API not found')
+
+
+def get_torrent(info_hash, timeout):
url = 'http://itorrents.org/torrent/{:X}.torrent'
req = request.Request(url.format(info_hash),
headers=pirate.data.default_headers)
req.add_header('Accept-encoding', 'gzip')
- torrent = request.urlopen(req, timeout=pirate.data.default_timeout)
+ torrent = request.urlopen(req, timeout=timeout)
if torrent.info().get('Content-Encoding') == 'gzip':
torrent = gzip.GzipFile(fileobj=BytesIO(torrent.read()))
return torrent.read()
-def save_torrents(printer, chosen_links, results, folder):
+def save_torrents(printer, chosen_links, results, folder, timeout):
for link in chosen_links:
- magnet = results[link]['magnet']
- name = re.search(r'dn=([^\&]*)', magnet)
- torrent_name = parse.unquote(name.group(1)).replace('+', ' ')
- info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16)
- torrent_name = torrent_name.replace('/', '_').replace('\\', '_')
+ result = results[link]
+ torrent_name = result['name'].replace('/', '_').replace('\\', '_')
file = os.path.join(folder, torrent_name + '.torrent')
try:
- torrent = get_torrent(info_hash)
+ torrent = get_torrent(result['info_hash'], timeout)
except urllib.error.HTTPError as e:
printer.print('There is no cached file for this torrent :('
' \nCode: {} - {}'.format(e.code, e.reason),
color='ERROR')
else:
open(file, 'wb').write(torrent)
- printer.print('Saved {:X} in {}'.format(info_hash, file))
+ printer.print('Saved {:X} in {}'.format(result['info_hash'], file))
def save_magnets(printer, chosen_links, results, folder):
for link in chosen_links:
- magnet = results[link]['magnet']
- name = re.search(r'dn=([^\&]*)', magnet)
- torrent_name = parse.unquote(name.group(1)).replace('+', ' ')
- info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16)
- torrent_name = torrent_name.replace('/', '_').replace('\\', '_')
+ result = results[link]
+ torrent_name = result['name'].replace('/', '_').replace('\\', '_')
file = os.path.join(folder, torrent_name + '.magnet')
- printer.print('Saved {:X} in {}'.format(info_hash, file))
+ printer.print('Saved {:X} in {}'.format(result['info_hash'], file))
with open(file, 'w') as f:
- f.write(magnet + '\n')
+ f.write(result['magnet'] + '\n')
def copy_magnets(printer, chosen_links, results):
clipboard_text = ''
for link in chosen_links:
- magnet = results[link]['magnet']
- info_hash = int(re.search(r'btih:([a-fA-F0-9]{40})', magnet).group(1), 16)
- clipboard_text += magnet + "\n"
- printer.print('Copying {:X} to clipboard'.format(info_hash))
+ result = results[link]
+ clipboard_text += result['magnet'] + "\n"
+ printer.print('Copying {:X} to clipboard'.format(result['info_hash']))
pyperclip.copy(clipboard_text)
diff --git a/setup.py b/setup.py
index 8330ca1..6ee0fdf 100755
--- a/setup.py
+++ b/setup.py
@@ -18,12 +18,11 @@ if __name__ == '__main__':
author_email='me@viktorstanchev.com',
license='AGPL',
packages=find_packages(),
- package_data={'': ["data/*.json"]},
+ package_data={'': ["data/*.json", "tests/data/*"]},
entry_points={
'console_scripts': ['pirate-get = pirate.pirate:main']
},
install_requires=['colorama>=0.3.3',
- 'beautifulsoup4>=4.4.1',
'veryprettytable>=0.8.1',
'pyperclip>=1.6.2'],
keywords=['torrent', 'magnet', 'download', 'tpb', 'client'],
diff --git a/tests/data/dan_bull_search.html b/tests/data/dan_bull_search.html
deleted file mode 100644
index b7799c8..0000000
--- a/tests/data/dan_bull_search.html
+++ /dev/null
@@ -1,461 +0,0 @@
-
-
-
-
The Pirate Bay - The galaxy's most resilient bittorrent site
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Search results: dan bull Displaying hits from 0 to 15 (approx 15 found)
-
-
-
-
-
-
-
-
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-  
- Uploaded 04-04 2014, Size 89.33 MiB, ULed by Capajebo
- |
- 16 |
- 1 |
-
-
-
-
- Audio
- (Other)
-
- |
-
-
-
- Uploaded 03-02 2014, Size 294 MiB, ULed by Vakume
- |
- 4 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 01-19 2013, Size 54.86 MiB, ULed by blowingfish
- |
- 2 |
- 0 |
-
-
-
-
- Audio
- (Other)
-
- |
-
-
-
- Uploaded 01-21 2010, Size 236.78 MiB, ULed by SuperSaru
- |
- 1 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 09-02 2014, Size 36.27 MiB, ULed by Bazookus
- |
- 1 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 09-27 2009, Size 5.51 MiB, ULed by oneanight
- |
- 0 |
- 1 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 11-29 2009, Size 5.07 MiB, ULed by epiclawl
- |
- 0 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 11-10 2011, Size 5.34 MiB, ULed by Imperator42
- |
- 0 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 12-20 2011, Size 4.8 MiB, ULed by lerdie
- |
- 0 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 12-21 2011, Size 3.4 MiB, ULed by mattdow
- |
- 0 |
- 1 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 12-21 2011, Size 4.8 MiB, ULed by lerdie
- |
- 0 |
- 1 |
-
-
-
-
- Audio
- (Other)
-
- |
-
-
-
- Uploaded 03-09 2012, Size 60.72 MiB, ULed by oneanight
- |
- 0 |
- 1 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 10-24 2012, Size 6.29 MiB, ULed by PIRATE300
- |
- 0 |
- 0 |
-
-
-
-
- Audio
- (Music)
-
- |
-
-
-
- Uploaded 11-10 2012, Size 6.41 MiB, ULed by AdpoX10
- |
- 0 |
- 0 |
-
-
-
-
- Audio
- (Other)
-
- |
-
-
-
- Uploaded 01-19 2013, Size 54.87 MiB, ULed by blowingfish
- |
- 0 |
- 1 |
-
-
-
-
-
-
