\s*
(.+?)(?=
)',
res, re.DOTALL).group(1)
@@ -101,10 +107,10 @@ def descriptions(chosen_links, mags, site, identifiers):
print(desc, color='zebra_0')
-def file_lists(chosen_links, mags, site, identifiers):
+def file_lists(chosen_links, results, site):
for link in chosen_links:
path = '/ajax_details_filelist.php'
- query = '?id=' + identifiers[int(link)]
+ query = '?id=' + results[link]['id']
req = request.Request(site + path + query,
headers=pirate.data.default_headers)
req.add_header('Accept-encoding', 'gzip')
@@ -113,10 +119,14 @@ def file_lists(chosen_links, mags, site, identifiers):
if f.info().get('Content-Encoding') == 'gzip':
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
+ # TODO: proper html decoding/parsing
res = f.read().decode('utf-8').replace(' ', ' ')
+ if 'File list not available.' in res:
+ print('File list not available.')
+ return
files = re.findall(r'
\s*([^<]+?)\s* | \s*([^<]+?)\s*', res)
- name = re.search(r'dn=([^\&]*)', mags[int(link)][0])
+ name = re.search(r'dn=([^\&]*)', results[link]['magnet'])
torrent_name = parse.unquote(name.group(1)).replace('+', ' ')
print('Files in "%s":' % torrent_name, color='zebra_1')
diff --git a/pirate/torrent.py b/pirate/torrent.py
index 6699b88..3c0aa16 100644
--- a/pirate/torrent.py
+++ b/pirate/torrent.py
@@ -6,57 +6,130 @@ import urllib.parse as parse
import urllib.error
import os.path
+from pyquery import PyQuery as pq
+
import pirate.data
from pirate.print import print
from io import BytesIO
-#todo: redo this with html parser instead of regex
-def remote(args, mirror):
+
+parser_regex = r'"(magnet\:\?xt=[^"]*)| | ([^<]+) | '
+
+
+def parse_category(category):
+ try:
+ category = int(category)
+ except ValueError:
+ pass
+ if category in pirate.data.categories.values():
+ return category
+ elif category in pirate.data.categories.keys():
+ return pirate.data.categories[category]
+ else:
+ print('Invalid category ignored', color='WARN')
+ return '0'
+
+
+def parse_sort(sort):
+ try:
+ sort = int(sort)
+ except ValueError:
+ pass
+ if sort in pirate.data.sorts.values():
+ return sort
+ elif sort in pirate.data.sorts.keys():
+ return pirate.data.sorts[sort]
+ else:
+ print('Invalid sort ignored', color='WARN')
+ return '99'
+
+
+#TODO: warn users when using a sort in a mode that doesn't accept sorts
+#TODO: warn users when using search terms in a mode that doesn't accept search terms
+#TODO: same with page parameter for top and top48h
+#TODO: warn the user if trying to use a minor category with top48h
+def build_request_path(page, category, sort, mode, terms):
+ if mode == 'browse':
+ if(category == 0):
+ category = 100
+ return '/browse/{}/{}/{}'.format(category, page, sort)
+ elif mode == 'recent':
+ # This is not a typo. There is no / between 48h and the category.
+ path = '/top/48h'
+ # only major categories can be used with this mode
+ if(category == 0):
+ return path + 'all'
+ else:
+ return path + str(category)
+ elif mode == 'top':
+ path = '/top/'
+ if(category == 0):
+ return path + 'all'
+ else:
+ return path + str(category)
+ elif mode == 'search':
+ query = urllib.parse.quote_plus(' '.join(terms))
+ return '/search/{}/{}/{}/{}'.format(query, page, sort, category)
+ else:
+ raise Exception('Unknown mode.')
+
+
+# this returns a list of dictionaries
+def parse_page(html):
+ d = pq(html)
+
+ results = []
+ # parse the rows one by one
+ for row in d('table#searchResult tr'):
+ drow = d(row)
+ if len(drow('th')) > 0:
+ continue
+
+ # grab info about the row
+ magnet = pq(drow(':eq(0)>td:nth-child(2)>a:nth-child(2)')[0]).attr('href')
+ seeds = pq(drow(':eq(0)>td:nth-child(3)')).text()
+ leechers = pq(drow(':eq(0)>td:nth-child(4)')).text()
+ id_ = pq(drow('.detLink')).attr('href').split('/')[2]
+
+ # parse descriptions separately
+ desc_text = pq(drow('font.detDesc')[0]).text()
+ size = re.findall(r'(?<=Size )[0-9.]+\s[KMGT]*[i ]*B', desc_text)[0].split()
+ uploaded = re.findall(r'(?<=Uploaded ).+(?=\, Size)', desc_text)[0]
+
+ results.append({
+ 'magnet': magnet,
+ 'seeds': seeds,
+ 'leechers': leechers,
+ 'size': size,
+ 'uploaded': uploaded,
+ 'id': id_
+ })
+
+ # check for a blocked mirror
+ no_results = re.search(r'No hits\. Try adding an asterisk in '
+ r'you search phrase\.', html)
+ if len(results) == 0 and no_results is None:
+ # Contradiction - we found no results,
+ # but the page didn't say there were no results.
+ # The page is probably not actually the pirate bay,
+ # so let's try another mirror
+ raise IOError('Blocked mirror detected.')
+
+ return results
+
+
+def remote(pages, category, sort, mode, terms, mirror):
res_l = []
- pages = int(args.pages)
+
if pages < 1:
raise ValueError('Please provide an integer greater than 0 '
'for the number of pages to fetch.')
- if str(args.category) in pirate.data.categories.values():
- category = args.category
- elif args.category in pirate.data.categories.keys():
- category = pirate.data.categories[args.category]
- else:
- category = '0'
- print('Invalid category ignored', color='WARN')
-
- if str(args.sort) in pirate.data.sorts.values():
- sort = args.sort
- elif args.sort in pirate.data.sorts.keys():
- sort = pirate.data.sorts[args.sort]
- else:
- sort = '99'
- print('Invalid sort ignored', color='WARN')
# Catch the Ctrl-C exception and exit cleanly
try:
- sizes = []
- uploaded = []
- identifiers = []
for page in range(pages):
- if args.browse:
- path = '/browse/'
- if(category == 0):
- category = 100
- path = '/browse/' + '/'.join(str(i) for i in (
- category, page, sort))
- elif len(args.search) == 0:
- path = '/top/48h' if args.recent else '/top/'
- if(category == 0):
- path += 'all'
- else:
- path += str(category)
- else:
- path = '/search/' + '/'.join(str(i) for i in (
- '+'.join(args.search),
- page, sort,
- category))
+ path = build_request_path(page, category, sort, mode, terms)
req = request.Request(mirror + path,
headers=pirate.data.default_headers)
@@ -65,53 +138,14 @@ def remote(args, mirror):
if f.info().get('Content-Encoding') == 'gzip':
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
res = f.read().decode('utf-8')
- found = re.findall(r'"(magnet\:\?xt=[^"]*)|
'
- r'([^<]+) | ', res)
- # check for a blocked mirror
- no_results = re.search(r'No hits\. Try adding an asterisk in '
- r'you search phrase\.', res)
- if found == [] and no_results is None:
- # Contradiction - we found no results,
- # but the page didn't say there were no results.
- # The page is probably not actually the pirate bay,
- # so let's try another mirror
- raise IOError('Blocked mirror detected.')
+ res_l += parse_page(res)
- # get sizes as well and substitute the character
- sizes.extend([match.replace(' ', ' ').split()
- for match in re.findall(r'(?<=Size )[0-9.]'
- r'+\ \;[KMGT]*[i ]*B', res)])
-
- uploaded.extend([match.replace(' ', ' ')
- for match in re.findall(r'(?<=Uploaded )'
- r'.+(?=\, Size)',res)])
-
- identifiers.extend([match.replace(' ', ' ')
- for match in re.findall('(?<=/torrent/)'
- '[0-9]+(?=/)',res)])
-
- state = 'seeds'
- curr = ['', 0, 0] #magnet, seeds, leeches
- for f in found:
- if f[1] == '':
- curr[0] = f[0]
- else:
- if state == 'seeds':
- curr[1] = f[1]
- state = 'leeches'
- else:
- curr[2] = f[1]
- state = 'seeds'
- res_l.append(curr)
- curr = ['', 0, 0]
- except KeyboardInterrupt :
+ except KeyboardInterrupt:
print('\nCancelled.')
sys.exit(0)
- # return the sizes in a spearate list
- return res_l, sizes, uploaded, identifiers
-
+ return res_l
def get_torrent(info_hash):
@@ -127,9 +161,9 @@ def get_torrent(info_hash):
return torrent.read()
-def save_torrents(chosen_links, mags, folder):
+def save_torrents(chosen_links, results, folder):
for link in chosen_links:
- magnet = mags[int(link)][0]
+ magnet = results[link]['magnet']
name = re.search(r'dn=([^\&]*)', magnet)
torrent_name = parse.unquote(name.group(1)).replace('+', ' ')
info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16)
@@ -146,7 +180,7 @@ def save_torrents(chosen_links, mags, folder):
def save_magnets(chosen_links, mags, folder):
for link in chosen_links:
- magnet = mags[int(link)][0]
+ magnet = results[link]['magnet']
name = re.search(r'dn=([^\&]*)', magnet)
torrent_name = parse.unquote(name.group(1)).replace('+', ' ')
info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16)
diff --git a/setup.py b/setup.py
index 8f8f6c1..42707ef 100755
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,7 @@ setup(name='pirate-get',
entry_points={
'console_scripts': ['pirate-get = pirate.pirate:main']
},
- install_requires=['colorama>=0.3.3'],
+ install_requires=['colorama>=0.3.3', 'pyquery>=1.2.9'],
keywords=['torrent', 'magnet', 'download', 'tpb', 'client'],
classifiers=[
'Topic :: Utilities',
diff --git a/tests/data/blocked.html b/tests/data/blocked.html
new file mode 100644
index 0000000..dd4bc42
--- /dev/null
+++ b/tests/data/blocked.html
@@ -0,0 +1 @@
+blocked.
diff --git a/tests/data/dan_bull_search.html b/tests/data/dan_bull_search.html
new file mode 100644
index 0000000..b7799c8
--- /dev/null
+++ b/tests/data/dan_bull_search.html
@@ -0,0 +1,461 @@
+
+
+
+
The Pirate Bay - The galaxy's most resilient bittorrent site
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Search results: dan bull Displaying hits from 0 to 15 (approx 15 found)
+
+
+
+
+
+
+
+
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+  
+ Uploaded 04-04 2014, Size 89.33 MiB, ULed by Capajebo
+ |
+ 16 |
+ 1 |
+
+
+
+
+ Audio
+ (Other)
+
+ |
+
+
+
+ Uploaded 03-02 2014, Size 294 MiB, ULed by Vakume
+ |
+ 4 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 01-19 2013, Size 54.86 MiB, ULed by blowingfish
+ |
+ 2 |
+ 0 |
+
+
+
+
+ Audio
+ (Other)
+
+ |
+
+
+
+ Uploaded 01-21 2010, Size 236.78 MiB, ULed by SuperSaru
+ |
+ 1 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 09-02 2014, Size 36.27 MiB, ULed by Bazookus
+ |
+ 1 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 09-27 2009, Size 5.51 MiB, ULed by oneanight
+ |
+ 0 |
+ 1 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 11-29 2009, Size 5.07 MiB, ULed by epiclawl
+ |
+ 0 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 11-10 2011, Size 5.34 MiB, ULed by Imperator42
+ |
+ 0 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 12-20 2011, Size 4.8 MiB, ULed by lerdie
+ |
+ 0 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 12-21 2011, Size 3.4 MiB, ULed by mattdow
+ |
+ 0 |
+ 1 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 12-21 2011, Size 4.8 MiB, ULed by lerdie
+ |
+ 0 |
+ 1 |
+
+
+
+
+ Audio
+ (Other)
+
+ |
+
+
+
+ Uploaded 03-09 2012, Size 60.72 MiB, ULed by oneanight
+ |
+ 0 |
+ 1 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 10-24 2012, Size 6.29 MiB, ULed by PIRATE300
+ |
+ 0 |
+ 0 |
+
+
+
+
+ Audio
+ (Music)
+
+ |
+
+
+
+ Uploaded 11-10 2012, Size 6.41 MiB, ULed by AdpoX10
+ |
+ 0 |
+ 0 |
+
+
+
+
+ Audio
+ (Other)
+
+ |
+
+
+
+ Uploaded 01-19 2013, Size 54.87 MiB, ULed by blowingfish
+ |
+ 0 |
+ 1 |
+
+
+
+
+
+
