From e7fb682309c5e84114ba47fc5fe9b5ffbce3fa75 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sun, 30 Aug 2015 01:26:54 +0000 Subject: [PATCH 01/13] New python .gitignore --- .gitignore | 60 +++- setup.py | 17 ++ src/pirate-get.py | 696 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 772 insertions(+), 1 deletion(-) create mode 100644 setup.py create mode 100755 src/pirate-get.py diff --git a/.gitignore b/.gitignore index ba31760..f0b338d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,59 @@ -pirate-get-* +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +.virtualenv + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..556dbde --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +setup(name='pirate-get', + version='0.2.4', + description='A command line interface for The Pirate Bay', + url='https://github.com/vikstrous/pirate-get', + author='vikstrous', + author_email='', + license='GPL', + packages=['pirate-get'], + entry_points={ + 'console_scripts': ['pirate-get = src.main:main'] + }, + keywords=['server'], + classifiers=[ + 'License :: OSI Approved :: GNU General Public License (GPL)', + ]) \ No newline at end of file diff --git a/src/pirate-get.py b/src/pirate-get.py new file mode 100755 index 0000000..4a9aeaa --- /dev/null +++ b/src/pirate-get.py @@ -0,0 +1,696 @@ +#!/usr/bin/env python +# +# Copyright 2015, Viktor Stanchev and contributors +# +# This file is part of pirate-get. +# +# pirate-get is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pirate-get is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with pirate-get. If not, see . + +import os +import sys +import re +import string +import gzip +import configparser +import argparse +import builtins +import subprocess + +import webbrowser +import urllib.request as request +import urllib.parse as parse + +from html.parser import HTMLParser +from urllib.error import URLError, HTTPError +from socket import timeout +from io import BytesIO +from os.path import expanduser, expandvars + +colored_output = True + +default_timeout = 10 + +default_headers = {'User-Agent': 'pirate get'} + +categories = { + 'All': 0, + 'Applications': 300, + 'Applications/Android': 306, + 'Applications/Handheld': 304, + 'Applications/IOS (iPad/iPhone)': 305, + 'Applications/Mac': 302, + 'Applications/Other OS': 399, + 'Applications/UNIX': 303, + 'Applications/Windows': 301, + 'Audio': 100, + 'Audio/Audio books': 102, + 'Audio/FLAC': 104, + 'Audio/Music': 101, + 'Audio/Other': 199, + 'Audio/Sound clips': 103, + 'Games': 400, + 'Games/Android': 408, + 'Games/Handheld': 406, + 'Games/IOS (iPad/iPhone)': 407, + 'Games/Mac': 402, + 'Games/Other': 499, + 'Games/PC': 401, + 'Games/PSx': 403, + 'Games/Wii': 405, + 'Games/XBOX360': 404, + 'Other': 600, + 'Other/Comics': 602, + 'Other/Covers': 604, + 'Other/E-books': 601, + 'Other/Other': 699, + 'Other/Physibles': 605, + 'Other/Pictures': 603, + 'Porn': 500, + 'Porn/Games': 504, + 'Porn/HD - Movies': 505, + 'Porn/Movie clips': 506, + 'Porn/Movies': 501, + 'Porn/Movies DVDR': 502, + 'Porn/Other': 599, + 'Porn/Pictures': 503, + 'Video': 200, + 'Video/3D': 209, + 'Video/HD - Movies': 207, + 'Video/HD - TV shows': 208, + 'Video/Handheld': 206, + 'Video/Movie clips': 204, + 'Video/Movies': 201, + 'Video/Movies DVDR': 202, + 'Video/Music videos': 203, + 'Video/Other': 299, + 'Video/TV shows': 205} + +sorts = { + 'TitleDsc': 1, 'TitleAsc': 2, + 'DateDsc': 3, 'DateAsc': 4, + 'SizeDsc': 5, 'SizeAsc': 6, + 'SeedersDsc': 7, 'SeedersAsc': 8, + 'LeechersDsc': 9, 'LeechersAsc': 10, + 'CategoryDsc': 13, 'CategoryAsc': 14, + 'Default': 99} + + +# create a subclass and override the handler methods +class BayParser(HTMLParser): + title = '' + q = '' + state = 'looking' + results = [] + + def __init__(self, q): + HTMLParser.__init__(self) + self.q = q.lower() + + def handle_starttag(self, tag, attrs): + if tag == 'title': + self.state = 'title' + if tag == 'magnet' and self.state == 'matched': + self.state = 'magnet' + + def handle_data(self, data): + if self.state == 'title': + if data.lower().find(self.q) != -1: + self.title = data + self.state = 'matched' + else: + self.state = 'looking' + if self.state == 'magnet': + self.results.append([ + 'magnet:?xt=urn:btih:' + + parse.quote(data) + + '&dn=' + + parse.quote(self.title), '?', '?']) + self.state = 'looking' + + +def print(*args, **kwargs): + if kwargs.get('color', False) and colored_output: + try: + import colorama + except (ImportError): + pass + else: + colorama.init() + color_dict = { + 'default': '', + 'header': colorama.Back.BLACK + colorama.Fore.WHITE, + 'alt': colorama.Fore.YELLOW, + 'zebra_0': '', + 'zebra_1': colorama.Fore.BLUE, + 'WARN': colorama.Fore.MAGENTA, + 'ERROR': colorama.Fore.RED} + + c = color_dict[kwargs.pop('color')] + args = (c + args[0],) + args[1:] + (colorama.Style.RESET_ALL,) + kwargs.pop('color', None) + return builtins.print(*args, **kwargs) + else: + kwargs.pop('color', None) + return builtins.print(*args, **kwargs) + + +def parse_cmd(cmd, url): + cmd_args_regex = r'''(('[^']*'|"[^"]*"|(\\\s|[^\s])+)+ *)''' + ret = re.findall(cmd_args_regex, cmd) + ret = [i[0].strip().replace('%s', url) for i in ret] + ret_no_quotes = [] + for item in ret: + if (item[0] == "'" and item[-1] == "'") or (item[0] == '"' and item[-1] == '"'): + ret_no_quotes.append(item[1:-1]) + else: + ret_no_quotes.append(item) + return ret_no_quotes + + +#todo: redo this with html parser instead of regex +def remote(args, mirror): + res_l = [] + pages = int(args.pages) + if pages < 1: + raise ValueError('Please provide an integer greater than 0 ' + 'for the number of pages to fetch.') + + if str(args.category) in categories.values(): + category = args.category + elif args.category in categories.keys(): + category = categories[args.category] + else: + category = '0' + print('Invalid category ignored', color='WARN') + + if str(args.sort) in sorts.values(): + sort = args.sort + elif args.sort in sorts.keys(): + sort = sorts[args.sort] + else: + sort = '99' + print('Invalid sort ignored', color='WARN') + # Catch the Ctrl-C exception and exit cleanly + try: + sizes = [] + uploaded = [] + identifiers = [] + for page in range(pages): + if args.browse: + path = '/browse/' + if(category == 0): + category = 100 + path = '/browse/' + '/'.join(str(i) for i in ( + category, page, sort)) + elif len(args.search) == 0: + path = '/top/48h' if args.recent else '/top/' + if(category == 0): + path += 'all' + else: + path += str(category) + else: + path = '/search/' + '/'.join(str(i) for i in ( + '+'.join(args.search), + page, sort, + category)) + + req = request.Request(mirror + path, headers=default_headers) + req.add_header('Accept-encoding', 'gzip') + f = request.urlopen(req, timeout=default_timeout) + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + res = f.read().decode('utf-8') + found = re.findall(r'"(magnet\:\?xt=[^"]*)|' + r'([^<]+)', res) + + # check for a blocked mirror + no_results = re.search(r'No hits\. Try adding an asterisk in ' + r'you search phrase\.', res) + if found == [] and no_results is None: + # Contradiction - we found no results, + # but the page didn't say there were no results. + # The page is probably not actually the pirate bay, + # so let's try another mirror + raise IOError('Blocked mirror detected.') + + # get sizes as well and substitute the   character + sizes.extend([match.replace(' ', ' ').split() + for match in re.findall(r'(?<=Size )[0-9.]' + r'+\ \;[KMGT]*[i ]*B', res)]) + + uploaded.extend([match.replace(' ', ' ') + for match in re.findall(r'(?<=Uploaded )' + r'.+(?=\, Size)',res)]) + + identifiers.extend([match.replace(' ', ' ') + for match in re.findall('(?<=/torrent/)' + '[0-9]+(?=/)',res)]) + + state = 'seeds' + curr = ['', 0, 0] #magnet, seeds, leeches + for f in found: + if f[1] == '': + curr[0] = f[0] + else: + if state == 'seeds': + curr[1] = f[1] + state = 'leeches' + else: + curr[2] = f[1] + state = 'seeds' + res_l.append(curr) + curr = ['', 0, 0] + except KeyboardInterrupt : + print('\nCancelled.') + sys.exit(0) + + # return the sizes in a spearate list + return res_l, sizes, uploaded, identifiers + + +def local(db, search): + xml = open(db).readlines() + parser = BayParser(' '.join(search)) + parser.feed(''.join(xml)) + return parser.results + + +def load_config(): + config = configparser.ConfigParser() + + # default options + config.add_section('Save') + config.set('Save', 'magnets', 'false') + config.set('Save', 'torrents', 'false') + config.set('Save', 'directory', os.getcwd()) + + config.add_section('LocalDB') + config.set('LocalDB', 'enabled', 'false') + config.set('LocalDB', 'path', expanduser('~/downloads/pirate-get/db')) + + config.add_section('Misc') + config.set('Misc', 'openCommand', '') + config.set('Misc', 'transmission', 'false') + config.set('Misc', 'colors', 'true') + + # user-defined config files + main = expandvars('$XDG_CONFIG_HOME/pirate-get') + alt = expanduser('~/.config/pirate-get') + + # read config file + config.read([main] if os.path.isfile(main) else [alt]) + + # expand env variables + directory = expanduser(expandvars(config.get('Save', 'Directory'))) + path = expanduser(expandvars(config.get('LocalDB', 'path'))) + + config.set('Save', 'Directory', directory) + config.set('LocalDB', 'path', path) + + return config + + +def get_torrent(info_hash): + url = 'http://torcache.net/torrent/{:X}.torrent' + req = request.Request(url.format(info_hash), headers=default_headers) + req.add_header('Accept-encoding', 'gzip') + + torrent = request.urlopen(req, timeout=default_timeout) + if torrent.info().get('Content-Encoding') == 'gzip': + torrent = gzip.GzipFile(fileobj=BytesIO(torrent.read())) + + return torrent.read() + + +def print_search_results(mags, sizes, uploaded, local): + columns = int(os.popen('stty size', 'r').read().split()[1]) + cur_color = 'zebra_0' + + if local: + print('{:>4} {:{length}}'.format( + 'LINK', 'NAME', length=columns - 8), + color='header') + else: + print('{:>4} {:>5} {:>5} {:>5} {:9} {:11} {:{length}}'.format( + 'LINK', 'SEED', 'LEECH', 'RATIO', + 'SIZE', 'UPLOAD', 'NAME', length=columns - 52), + color='header') + + for m, magnet in enumerate(mags): + # Alternate between colors + cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' + + name = re.search(r'dn=([^\&]*)', magnet[0]) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + + if local: + line = '{:5} {:{length}}' + content = [m, torrent_name[:columns]] + else: + no_seeders, no_leechers = map(int, magnet[1:]) + size, unit = (float(sizes[m][0]), sizes[m][1]) if sizes else (0, '???') + date = uploaded[m] + + # compute the S/L ratio (Higher is better) + try: + ratio = no_seeders / no_leechers + except ZeroDivisionError: + ratio = float('inf') + + line = ('{:4} {:5} {:5} {:5.1f} {:5.1f}' + ' {:3} {:<11} {:{length}}') + content = [m, no_seeders, no_leechers, ratio, + size, unit, date, torrent_name[:columns - 52]] + + # enhanced print output with justified columns + print(line.format(*content, length=columns - 52), color=cur_color) + + +def print_descriptions(chosen_links, mags, site, identifiers): + for link in chosen_links: + link = int(link) + path = '/torrent/%s/' % identifiers[link] + req = request.Request(site + path, headers=default_headers) + req.add_header('Accept-encoding', 'gzip') + f = request.urlopen(req, timeout=default_timeout) + + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + + res = f.read().decode('utf-8') + name = re.search(r'dn=([^\&]*)', mags[link][0]) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + desc = re.search(r'
\s*
(.+?)(?=
)', + res, re.DOTALL).group(1) + + # Replace HTML links with markdown style versions + desc = re.sub(r']*>(\s*)([^<]+?)(\s*' + r')', r'\2[\3](\1)\4', desc) + + print('Description for "%s":' % torrent_name, color='zebra_1') + print(desc, color='zebra_0') + + +def print_file_lists(chosen_links, mags, site, identifiers): + for link in chosen_links: + path = '/ajax_details_filelist.php' + query = '?id=' + identifiers[int(link)] + req = request.Request(site + path + query, headers=default_headers) + req.add_header('Accept-encoding', 'gzip') + f = request.urlopen(req, timeout=default_timeout) + + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + + res = f.read().decode('utf-8').replace(' ', ' ') + files = re.findall(r'\s*([^<]+?)\s*\s*([^<]+?)\s*', res) + name = re.search(r'dn=([^\&]*)', mags[int(link)][0]) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + + print('Files in "%s":' % torrent_name, color='zebra_1') + cur_color = 'zebra_0' + + for f in files: + print('{0[0]:>11} {0[1]}'.format(f), color=cur_color) + cur_color = 'zebra_0' if (cur_color == 'zebra_1') else 'zebra_1' + + +def save_torrents(chosen_links, mags, folder): + for link in chosen_links: + magnet = mags[int(link)][0] + name = re.search(r'dn=([^\&]*)', magnet) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) + file = os.path.join(folder, torrent_name + '.torrent') + + try: + torrent = get_torrent(info_hash) + except HTTPError: + print('There is no cached file for this torrent :(', color='ERROR') + else: + open(file,'wb').write(torrent) + print('Saved {:X} in {}'.format(info_hash, file)) + + +def save_magnets(chosen_links, mags, folder): + for link in chosen_links: + magnet = mags[int(link)][0] + name = re.search(r'dn=([^\&]*)', magnet) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) + file = os.path.join(folder, torrent_name + '.magnet') + + print('Saved {:X} in {}'.format(info_hash, file)) + with open(file, 'w') as f: + f.write(magnet + '\n') + + +def main(): + config = load_config() + + parser = argparse.ArgumentParser( + description='finds and downloads torrents from the Pirate Bay') + parser.add_argument('-b', dest='browse', + action='store_true', + help='display in Browse mode') + parser.add_argument('search', metavar='search', + nargs='*', help='term to search for') + parser.add_argument('-c', dest='category', metavar='category', + help='specify a category to search', default='All') + parser.add_argument('-s', dest='sort', metavar='sort', + help='specify a sort option', default='SeedersDsc') + parser.add_argument('-R', dest='recent', action='store_true', + help='torrents uploaded in the last 48hours.' + '*ignored in searches*') + parser.add_argument('-l', dest='list_categories', + action='store_true', + help='list categories') + parser.add_argument('--list_sorts', dest='list_sorts', + action='store_true', + help='list Sortable Types') + parser.add_argument('-L', '--local', dest='database', + help='an xml file containing the Pirate Bay database') + parser.add_argument('-p', dest='pages', default=1, + help='the number of pages to fetch ' + "(doesn't work with --local)") + parser.add_argument('-0', dest='first', + action='store_true', + help='choose the top result') + parser.add_argument('-a', '--download-all', + action='store_true', + help='download all results') + parser.add_argument('-t', '--transmission', + action='store_true', + help='open magnets with transmission-remote') + parser.add_argument('-P', '--port', dest='port', + help='transmission-remote rpc port. default is 9091') + parser.add_argument('-C', '--custom', dest='command', + help='open magnets with a custom command' + ' (%%s will be replaced with the url)') + parser.add_argument('-M', '--save-magnets', + action='store_true', + help='save magnets links as files') + parser.add_argument('-T', '--save-torrents', + action='store_true', + help='save torrent files') + parser.add_argument('-S', '--save-directory', + type=str, metavar='DIRECTORY', + help='directory where to save downloaded files' + ' (if none is given $PWD will be used)') + parser.add_argument('--disable-colors', dest='color', + action='store_false', + help='disable colored output') + args = parser.parse_args() + + if (config.getboolean('Misc', 'colors') and not args.color + or not config.getboolean('Misc', 'colors')): + global colored_output + colored_output = False + + if args.save_directory: + config.set('Save', 'directory', args.save_directory) + + transmission_command = ['transmission-remote'] + if args.port: + transmission_command.append(args.port) + + if args.transmission or config.getboolean('Misc', 'transmission'): + ret = subprocess.call(transmission_command + ['-l'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if ret != 0: + print('Transmission is not running.') + return + + if args.list_categories: + cur_color = 'zebra_0' + for key, value in sorted(categories.items()) : + cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' + print(str(value), '\t', key, sep='', color=cur_color) + return + + if args.list_sorts: + cur_color = 'zebra_0' + for key, value in sorted(sorts.items()): + cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' + print(str(value), '\t', key, sep='', color=cur_color) + return + + if args.database or config.getboolean('LocalDB', 'enabled'): + if args.database: + path = args.database + else: + path = config.get('LocalDB', 'path') + mags = local(path, args.search) + sizes, uploaded = [], [] + + else: + mags, mirrors = [], {'https://thepiratebay.se'} + try: + req = request.Request('https://proxybay.co/list.txt', + headers=default_headers) + f = request.urlopen(req, timeout=default_timeout) + except IOError: + print('Could not fetch additional mirrors', color='WARN') + else: + if f.getcode() != 200: + raise IOError('The proxy bay responded with an error.') + mirrors = mirrors.union([i.decode('utf-8').strip() + for i in f.readlines()][3:]) + + for mirror in mirrors: + try: + print('Trying', mirror, end='... ') + mags, sizes, uploaded, identifiers = remote(args, mirror) + except (URLError, IOError, ValueError, timeout): + print('Failed', color='WARN') + else: + site = mirror + print('Ok', color='alt') + break + else: + print('No available mirrors :(', color='WARN') + return + + if not mags: + print('No results') + return + + print_search_results(mags, sizes, uploaded, local=args.database) + + if args.first: + print('Choosing first result') + choices = [0] + elif args.download_all: + print('Downloading all results') + choices = range(len(mags)) + else: + # New input loop to support different link options + while True: + print("\nSelect links (Type 'h' for more options" + ", 'q' to quit)", end='\b', color='alt') + try: + l=input(': ') + except KeyboardInterrupt : + print('\nCancelled.') + return + + try: + # Very permissive handling + # Check for any occurances or d, f, p, t, m, or q + cmd_code_match = re.search(r'([hdfpmtq])', l, + flags=re.IGNORECASE) + if cmd_code_match: + code = cmd_code_match.group(0).lower() + else: + code = None + + # Clean up command codes + # Substitute multiple consecutive spaces/commas for single + # comma remove anything that isn't an integer or comma. + # Turn into list + l = re.sub(r'^[hdfp, ]*|[hdfp, ]*$', '', l) + l = re.sub('[ ,]+', ',', l) + l = re.sub('[^0-9,]', '', l) + choices = l.split(',') + + # Act on option, if supplied + print('') + if code == 'h': + print('Options:', + ': Download selected torrents', + '[m]: Save magnets as files', + '[t]: Save .torrent files', + '[d]: Get descriptions', + '[f]: Get files', + '[p] Print search results', + '[q] Quit', sep='\n') + elif code == 'q': + print('Bye.', color='alt') + return + elif code == 'd': + print_descriptions(choices, mags, site, identifiers) + elif code == 'f': + print_file_lists(choices, mags, site, identifiers) + elif code == 'p': + print_search_results(mags, sizes, uploaded) + elif code == 'm': + save_magnets(choices, mags, + config.get('Save', 'directory')) + elif code == 't': + save_torrents(choices, mags, + config.get('Save', 'directory')) + elif not l: + print('No links entered!', color='WARN') + else: + break + except Exception as e: + print('Exception:', e, color='ERROR') + choices = () + + save_to_file = False + + if args.save_magnets or config.getboolean('Save', 'magnets'): + print('Saving selected magnets...') + save_magnets(choices, mags, config.get('Save', 'directory')) + save_to_file = True + + if args.save_torrents or config.getboolean('Save', 'torrents'): + print('Saving selected torrents...') + save_torrents(choices, mags, config.get('Save', 'directory')) + save_to_file = True + + if save_to_file: + return + + for choice in choices: + url = mags[int(choice)][0] + + if args.transmission or config.getboolean('Misc', 'transmission'): + subprocess.call(transmission_command + ['-l', '--add', url], shell=False) + subprocess.call(transmission_command + ['-l']) + + elif args.command or config.get('Misc', 'openCommand'): + command = config.get('Misc', 'openCommand') + if args.command: + command = args.command + subprocess.call(parse_cmd(command, url), shell=False) + + else: + webbrowser.open(url) + + +if __name__ == '__main__': + main() From 6db3fa158b0f9e4fd64d9d3a40333d2d5bd4c7e6 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sun, 30 Aug 2015 01:27:24 +0000 Subject: [PATCH 02/13] Remove install/uninstall scripts --- install | 23 ----------------------- uninstall | 2 -- 2 files changed, 25 deletions(-) delete mode 100755 install delete mode 100755 uninstall diff --git a/install b/install deleted file mode 100755 index 10ad915..0000000 --- a/install +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -set -e - -TMP=$(mktemp pirate-get-XXXXXX) - -{ - if [ $(which python3) ] - then - python='python3' - else - python='python' - fi - - echo "#!/usr/bin/env $python" > "$TMP" && - - sed 1d $(dirname $0)/pirate-get.py >> "$TMP" - - cp "$TMP" /usr/bin/pirate-get && - chmod +x /usr/bin/pirate-get && - chmod 755 /usr/bin/pirate-get && - - rm $TMP -} || rm $TMP diff --git a/uninstall b/uninstall deleted file mode 100755 index 34edb8b..0000000 --- a/uninstall +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -rm /usr/bin/pirate-get From 442dc84cdfddaf133e118c3aeba8793081b70d39 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sun, 30 Aug 2015 01:28:43 +0000 Subject: [PATCH 03/13] Organize in modules --- data/categories.json | 53 ++++ data/sorts.json | 15 + pirate-get.py | 696 ------------------------------------------- pirate/__init__.py | 0 pirate/data.py | 8 + pirate/local.py | 41 +++ pirate/pirate.py | 308 +++++++++++++++++++ pirate/print.py | 126 ++++++++ pirate/torrent.py | 155 ++++++++++ src/pirate-get.py | 696 ------------------------------------------- 10 files changed, 706 insertions(+), 1392 deletions(-) create mode 100644 data/categories.json create mode 100644 data/sorts.json delete mode 100755 pirate-get.py create mode 100644 pirate/__init__.py create mode 100644 pirate/data.py create mode 100644 pirate/local.py create mode 100755 pirate/pirate.py create mode 100644 pirate/print.py create mode 100644 pirate/torrent.py delete mode 100755 src/pirate-get.py diff --git a/data/categories.json b/data/categories.json new file mode 100644 index 0000000..2f53287 --- /dev/null +++ b/data/categories.json @@ -0,0 +1,53 @@ +{ +"All": 0, +"Applications": 300, +"Applications/Android": 306, +"Applications/Handheld": 304, +"Applications/IOS (iPad/iPhone)": 305, +"Applications/Mac": 302, +"Applications/Other OS": 399, +"Applications/UNIX": 303, +"Applications/Windows": 301, +"Audio": 100, +"Audio/Audio books": 102, +"Audio/FLAC": 104, +"Audio/Music": 101, +"Audio/Other": 199, +"Audio/Sound clips": 103, +"Games": 400, +"Games/Android": 408, +"Games/Handheld": 406, +"Games/IOS (iPad/iPhone)": 407, +"Games/Mac": 402, +"Games/Other": 499, +"Games/PC": 401, +"Games/PSx": 403, +"Games/Wii": 405, +"Games/XBOX360": 404, +"Other": 600, +"Other/Comics": 602, +"Other/Covers": 604, +"Other/E-books": 601, +"Other/Other": 699, +"Other/Physibles": 605, +"Other/Pictures": 603, +"Porn": 500, +"Porn/Games": 504, +"Porn/HD - Movies": 505, +"Porn/Movie clips": 506, +"Porn/Movies": 501, +"Porn/Movies DVDR": 502, +"Porn/Other": 599, +"Porn/Pictures": 503, +"Video": 200, +"Video/3D": 209, +"Video/HD - Movies": 207, +"Video/HD - TV shows": 208, +"Video/Handheld": 206, +"Video/Movie clips": 204, +"Video/Movies": 201, +"Video/Movies DVDR": 202, +"Video/Music videos": 203, +"Video/Other": 299, +"Video/TV shows": 205 +} diff --git a/data/sorts.json b/data/sorts.json new file mode 100644 index 0000000..d5f5ce1 --- /dev/null +++ b/data/sorts.json @@ -0,0 +1,15 @@ +{ +"TitleDsc": 1, +"TitleAsc": 2, +"DateDsc": 3, +"DateAsc": 4, +"SizeDsc": 5, +"SizeAsc": 6, +"SeedersDsc": 7, +"SeedersAsc": 8, +"LeechersDsc": 9, +"LeechersAsc": 10, +"CategoryDsc": 13, +"CategoryAsc": 14, +"Default": 99 +} \ No newline at end of file diff --git a/pirate-get.py b/pirate-get.py deleted file mode 100755 index 4a9aeaa..0000000 --- a/pirate-get.py +++ /dev/null @@ -1,696 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2015, Viktor Stanchev and contributors -# -# This file is part of pirate-get. -# -# pirate-get is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# pirate-get is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with pirate-get. If not, see . - -import os -import sys -import re -import string -import gzip -import configparser -import argparse -import builtins -import subprocess - -import webbrowser -import urllib.request as request -import urllib.parse as parse - -from html.parser import HTMLParser -from urllib.error import URLError, HTTPError -from socket import timeout -from io import BytesIO -from os.path import expanduser, expandvars - -colored_output = True - -default_timeout = 10 - -default_headers = {'User-Agent': 'pirate get'} - -categories = { - 'All': 0, - 'Applications': 300, - 'Applications/Android': 306, - 'Applications/Handheld': 304, - 'Applications/IOS (iPad/iPhone)': 305, - 'Applications/Mac': 302, - 'Applications/Other OS': 399, - 'Applications/UNIX': 303, - 'Applications/Windows': 301, - 'Audio': 100, - 'Audio/Audio books': 102, - 'Audio/FLAC': 104, - 'Audio/Music': 101, - 'Audio/Other': 199, - 'Audio/Sound clips': 103, - 'Games': 400, - 'Games/Android': 408, - 'Games/Handheld': 406, - 'Games/IOS (iPad/iPhone)': 407, - 'Games/Mac': 402, - 'Games/Other': 499, - 'Games/PC': 401, - 'Games/PSx': 403, - 'Games/Wii': 405, - 'Games/XBOX360': 404, - 'Other': 600, - 'Other/Comics': 602, - 'Other/Covers': 604, - 'Other/E-books': 601, - 'Other/Other': 699, - 'Other/Physibles': 605, - 'Other/Pictures': 603, - 'Porn': 500, - 'Porn/Games': 504, - 'Porn/HD - Movies': 505, - 'Porn/Movie clips': 506, - 'Porn/Movies': 501, - 'Porn/Movies DVDR': 502, - 'Porn/Other': 599, - 'Porn/Pictures': 503, - 'Video': 200, - 'Video/3D': 209, - 'Video/HD - Movies': 207, - 'Video/HD - TV shows': 208, - 'Video/Handheld': 206, - 'Video/Movie clips': 204, - 'Video/Movies': 201, - 'Video/Movies DVDR': 202, - 'Video/Music videos': 203, - 'Video/Other': 299, - 'Video/TV shows': 205} - -sorts = { - 'TitleDsc': 1, 'TitleAsc': 2, - 'DateDsc': 3, 'DateAsc': 4, - 'SizeDsc': 5, 'SizeAsc': 6, - 'SeedersDsc': 7, 'SeedersAsc': 8, - 'LeechersDsc': 9, 'LeechersAsc': 10, - 'CategoryDsc': 13, 'CategoryAsc': 14, - 'Default': 99} - - -# create a subclass and override the handler methods -class BayParser(HTMLParser): - title = '' - q = '' - state = 'looking' - results = [] - - def __init__(self, q): - HTMLParser.__init__(self) - self.q = q.lower() - - def handle_starttag(self, tag, attrs): - if tag == 'title': - self.state = 'title' - if tag == 'magnet' and self.state == 'matched': - self.state = 'magnet' - - def handle_data(self, data): - if self.state == 'title': - if data.lower().find(self.q) != -1: - self.title = data - self.state = 'matched' - else: - self.state = 'looking' - if self.state == 'magnet': - self.results.append([ - 'magnet:?xt=urn:btih:' + - parse.quote(data) + - '&dn=' + - parse.quote(self.title), '?', '?']) - self.state = 'looking' - - -def print(*args, **kwargs): - if kwargs.get('color', False) and colored_output: - try: - import colorama - except (ImportError): - pass - else: - colorama.init() - color_dict = { - 'default': '', - 'header': colorama.Back.BLACK + colorama.Fore.WHITE, - 'alt': colorama.Fore.YELLOW, - 'zebra_0': '', - 'zebra_1': colorama.Fore.BLUE, - 'WARN': colorama.Fore.MAGENTA, - 'ERROR': colorama.Fore.RED} - - c = color_dict[kwargs.pop('color')] - args = (c + args[0],) + args[1:] + (colorama.Style.RESET_ALL,) - kwargs.pop('color', None) - return builtins.print(*args, **kwargs) - else: - kwargs.pop('color', None) - return builtins.print(*args, **kwargs) - - -def parse_cmd(cmd, url): - cmd_args_regex = r'''(('[^']*'|"[^"]*"|(\\\s|[^\s])+)+ *)''' - ret = re.findall(cmd_args_regex, cmd) - ret = [i[0].strip().replace('%s', url) for i in ret] - ret_no_quotes = [] - for item in ret: - if (item[0] == "'" and item[-1] == "'") or (item[0] == '"' and item[-1] == '"'): - ret_no_quotes.append(item[1:-1]) - else: - ret_no_quotes.append(item) - return ret_no_quotes - - -#todo: redo this with html parser instead of regex -def remote(args, mirror): - res_l = [] - pages = int(args.pages) - if pages < 1: - raise ValueError('Please provide an integer greater than 0 ' - 'for the number of pages to fetch.') - - if str(args.category) in categories.values(): - category = args.category - elif args.category in categories.keys(): - category = categories[args.category] - else: - category = '0' - print('Invalid category ignored', color='WARN') - - if str(args.sort) in sorts.values(): - sort = args.sort - elif args.sort in sorts.keys(): - sort = sorts[args.sort] - else: - sort = '99' - print('Invalid sort ignored', color='WARN') - # Catch the Ctrl-C exception and exit cleanly - try: - sizes = [] - uploaded = [] - identifiers = [] - for page in range(pages): - if args.browse: - path = '/browse/' - if(category == 0): - category = 100 - path = '/browse/' + '/'.join(str(i) for i in ( - category, page, sort)) - elif len(args.search) == 0: - path = '/top/48h' if args.recent else '/top/' - if(category == 0): - path += 'all' - else: - path += str(category) - else: - path = '/search/' + '/'.join(str(i) for i in ( - '+'.join(args.search), - page, sort, - category)) - - req = request.Request(mirror + path, headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - res = f.read().decode('utf-8') - found = re.findall(r'"(magnet\:\?xt=[^"]*)|' - r'([^<]+)', res) - - # check for a blocked mirror - no_results = re.search(r'No hits\. Try adding an asterisk in ' - r'you search phrase\.', res) - if found == [] and no_results is None: - # Contradiction - we found no results, - # but the page didn't say there were no results. - # The page is probably not actually the pirate bay, - # so let's try another mirror - raise IOError('Blocked mirror detected.') - - # get sizes as well and substitute the   character - sizes.extend([match.replace(' ', ' ').split() - for match in re.findall(r'(?<=Size )[0-9.]' - r'+\ \;[KMGT]*[i ]*B', res)]) - - uploaded.extend([match.replace(' ', ' ') - for match in re.findall(r'(?<=Uploaded )' - r'.+(?=\, Size)',res)]) - - identifiers.extend([match.replace(' ', ' ') - for match in re.findall('(?<=/torrent/)' - '[0-9]+(?=/)',res)]) - - state = 'seeds' - curr = ['', 0, 0] #magnet, seeds, leeches - for f in found: - if f[1] == '': - curr[0] = f[0] - else: - if state == 'seeds': - curr[1] = f[1] - state = 'leeches' - else: - curr[2] = f[1] - state = 'seeds' - res_l.append(curr) - curr = ['', 0, 0] - except KeyboardInterrupt : - print('\nCancelled.') - sys.exit(0) - - # return the sizes in a spearate list - return res_l, sizes, uploaded, identifiers - - -def local(db, search): - xml = open(db).readlines() - parser = BayParser(' '.join(search)) - parser.feed(''.join(xml)) - return parser.results - - -def load_config(): - config = configparser.ConfigParser() - - # default options - config.add_section('Save') - config.set('Save', 'magnets', 'false') - config.set('Save', 'torrents', 'false') - config.set('Save', 'directory', os.getcwd()) - - config.add_section('LocalDB') - config.set('LocalDB', 'enabled', 'false') - config.set('LocalDB', 'path', expanduser('~/downloads/pirate-get/db')) - - config.add_section('Misc') - config.set('Misc', 'openCommand', '') - config.set('Misc', 'transmission', 'false') - config.set('Misc', 'colors', 'true') - - # user-defined config files - main = expandvars('$XDG_CONFIG_HOME/pirate-get') - alt = expanduser('~/.config/pirate-get') - - # read config file - config.read([main] if os.path.isfile(main) else [alt]) - - # expand env variables - directory = expanduser(expandvars(config.get('Save', 'Directory'))) - path = expanduser(expandvars(config.get('LocalDB', 'path'))) - - config.set('Save', 'Directory', directory) - config.set('LocalDB', 'path', path) - - return config - - -def get_torrent(info_hash): - url = 'http://torcache.net/torrent/{:X}.torrent' - req = request.Request(url.format(info_hash), headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - - torrent = request.urlopen(req, timeout=default_timeout) - if torrent.info().get('Content-Encoding') == 'gzip': - torrent = gzip.GzipFile(fileobj=BytesIO(torrent.read())) - - return torrent.read() - - -def print_search_results(mags, sizes, uploaded, local): - columns = int(os.popen('stty size', 'r').read().split()[1]) - cur_color = 'zebra_0' - - if local: - print('{:>4} {:{length}}'.format( - 'LINK', 'NAME', length=columns - 8), - color='header') - else: - print('{:>4} {:>5} {:>5} {:>5} {:9} {:11} {:{length}}'.format( - 'LINK', 'SEED', 'LEECH', 'RATIO', - 'SIZE', 'UPLOAD', 'NAME', length=columns - 52), - color='header') - - for m, magnet in enumerate(mags): - # Alternate between colors - cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' - - name = re.search(r'dn=([^\&]*)', magnet[0]) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - - if local: - line = '{:5} {:{length}}' - content = [m, torrent_name[:columns]] - else: - no_seeders, no_leechers = map(int, magnet[1:]) - size, unit = (float(sizes[m][0]), sizes[m][1]) if sizes else (0, '???') - date = uploaded[m] - - # compute the S/L ratio (Higher is better) - try: - ratio = no_seeders / no_leechers - except ZeroDivisionError: - ratio = float('inf') - - line = ('{:4} {:5} {:5} {:5.1f} {:5.1f}' - ' {:3} {:<11} {:{length}}') - content = [m, no_seeders, no_leechers, ratio, - size, unit, date, torrent_name[:columns - 52]] - - # enhanced print output with justified columns - print(line.format(*content, length=columns - 52), color=cur_color) - - -def print_descriptions(chosen_links, mags, site, identifiers): - for link in chosen_links: - link = int(link) - path = '/torrent/%s/' % identifiers[link] - req = request.Request(site + path, headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) - - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - - res = f.read().decode('utf-8') - name = re.search(r'dn=([^\&]*)', mags[link][0]) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - desc = re.search(r'
\s*
(.+?)(?=
)', - res, re.DOTALL).group(1) - - # Replace HTML links with markdown style versions - desc = re.sub(r']*>(\s*)([^<]+?)(\s*' - r')', r'\2[\3](\1)\4', desc) - - print('Description for "%s":' % torrent_name, color='zebra_1') - print(desc, color='zebra_0') - - -def print_file_lists(chosen_links, mags, site, identifiers): - for link in chosen_links: - path = '/ajax_details_filelist.php' - query = '?id=' + identifiers[int(link)] - req = request.Request(site + path + query, headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) - - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - - res = f.read().decode('utf-8').replace(' ', ' ') - files = re.findall(r'\s*([^<]+?)\s*\s*([^<]+?)\s*', res) - name = re.search(r'dn=([^\&]*)', mags[int(link)][0]) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - - print('Files in "%s":' % torrent_name, color='zebra_1') - cur_color = 'zebra_0' - - for f in files: - print('{0[0]:>11} {0[1]}'.format(f), color=cur_color) - cur_color = 'zebra_0' if (cur_color == 'zebra_1') else 'zebra_1' - - -def save_torrents(chosen_links, mags, folder): - for link in chosen_links: - magnet = mags[int(link)][0] - name = re.search(r'dn=([^\&]*)', magnet) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) - file = os.path.join(folder, torrent_name + '.torrent') - - try: - torrent = get_torrent(info_hash) - except HTTPError: - print('There is no cached file for this torrent :(', color='ERROR') - else: - open(file,'wb').write(torrent) - print('Saved {:X} in {}'.format(info_hash, file)) - - -def save_magnets(chosen_links, mags, folder): - for link in chosen_links: - magnet = mags[int(link)][0] - name = re.search(r'dn=([^\&]*)', magnet) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) - file = os.path.join(folder, torrent_name + '.magnet') - - print('Saved {:X} in {}'.format(info_hash, file)) - with open(file, 'w') as f: - f.write(magnet + '\n') - - -def main(): - config = load_config() - - parser = argparse.ArgumentParser( - description='finds and downloads torrents from the Pirate Bay') - parser.add_argument('-b', dest='browse', - action='store_true', - help='display in Browse mode') - parser.add_argument('search', metavar='search', - nargs='*', help='term to search for') - parser.add_argument('-c', dest='category', metavar='category', - help='specify a category to search', default='All') - parser.add_argument('-s', dest='sort', metavar='sort', - help='specify a sort option', default='SeedersDsc') - parser.add_argument('-R', dest='recent', action='store_true', - help='torrents uploaded in the last 48hours.' - '*ignored in searches*') - parser.add_argument('-l', dest='list_categories', - action='store_true', - help='list categories') - parser.add_argument('--list_sorts', dest='list_sorts', - action='store_true', - help='list Sortable Types') - parser.add_argument('-L', '--local', dest='database', - help='an xml file containing the Pirate Bay database') - parser.add_argument('-p', dest='pages', default=1, - help='the number of pages to fetch ' - "(doesn't work with --local)") - parser.add_argument('-0', dest='first', - action='store_true', - help='choose the top result') - parser.add_argument('-a', '--download-all', - action='store_true', - help='download all results') - parser.add_argument('-t', '--transmission', - action='store_true', - help='open magnets with transmission-remote') - parser.add_argument('-P', '--port', dest='port', - help='transmission-remote rpc port. default is 9091') - parser.add_argument('-C', '--custom', dest='command', - help='open magnets with a custom command' - ' (%%s will be replaced with the url)') - parser.add_argument('-M', '--save-magnets', - action='store_true', - help='save magnets links as files') - parser.add_argument('-T', '--save-torrents', - action='store_true', - help='save torrent files') - parser.add_argument('-S', '--save-directory', - type=str, metavar='DIRECTORY', - help='directory where to save downloaded files' - ' (if none is given $PWD will be used)') - parser.add_argument('--disable-colors', dest='color', - action='store_false', - help='disable colored output') - args = parser.parse_args() - - if (config.getboolean('Misc', 'colors') and not args.color - or not config.getboolean('Misc', 'colors')): - global colored_output - colored_output = False - - if args.save_directory: - config.set('Save', 'directory', args.save_directory) - - transmission_command = ['transmission-remote'] - if args.port: - transmission_command.append(args.port) - - if args.transmission or config.getboolean('Misc', 'transmission'): - ret = subprocess.call(transmission_command + ['-l'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - if ret != 0: - print('Transmission is not running.') - return - - if args.list_categories: - cur_color = 'zebra_0' - for key, value in sorted(categories.items()) : - cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' - print(str(value), '\t', key, sep='', color=cur_color) - return - - if args.list_sorts: - cur_color = 'zebra_0' - for key, value in sorted(sorts.items()): - cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' - print(str(value), '\t', key, sep='', color=cur_color) - return - - if args.database or config.getboolean('LocalDB', 'enabled'): - if args.database: - path = args.database - else: - path = config.get('LocalDB', 'path') - mags = local(path, args.search) - sizes, uploaded = [], [] - - else: - mags, mirrors = [], {'https://thepiratebay.se'} - try: - req = request.Request('https://proxybay.co/list.txt', - headers=default_headers) - f = request.urlopen(req, timeout=default_timeout) - except IOError: - print('Could not fetch additional mirrors', color='WARN') - else: - if f.getcode() != 200: - raise IOError('The proxy bay responded with an error.') - mirrors = mirrors.union([i.decode('utf-8').strip() - for i in f.readlines()][3:]) - - for mirror in mirrors: - try: - print('Trying', mirror, end='... ') - mags, sizes, uploaded, identifiers = remote(args, mirror) - except (URLError, IOError, ValueError, timeout): - print('Failed', color='WARN') - else: - site = mirror - print('Ok', color='alt') - break - else: - print('No available mirrors :(', color='WARN') - return - - if not mags: - print('No results') - return - - print_search_results(mags, sizes, uploaded, local=args.database) - - if args.first: - print('Choosing first result') - choices = [0] - elif args.download_all: - print('Downloading all results') - choices = range(len(mags)) - else: - # New input loop to support different link options - while True: - print("\nSelect links (Type 'h' for more options" - ", 'q' to quit)", end='\b', color='alt') - try: - l=input(': ') - except KeyboardInterrupt : - print('\nCancelled.') - return - - try: - # Very permissive handling - # Check for any occurances or d, f, p, t, m, or q - cmd_code_match = re.search(r'([hdfpmtq])', l, - flags=re.IGNORECASE) - if cmd_code_match: - code = cmd_code_match.group(0).lower() - else: - code = None - - # Clean up command codes - # Substitute multiple consecutive spaces/commas for single - # comma remove anything that isn't an integer or comma. - # Turn into list - l = re.sub(r'^[hdfp, ]*|[hdfp, ]*$', '', l) - l = re.sub('[ ,]+', ',', l) - l = re.sub('[^0-9,]', '', l) - choices = l.split(',') - - # Act on option, if supplied - print('') - if code == 'h': - print('Options:', - ': Download selected torrents', - '[m]: Save magnets as files', - '[t]: Save .torrent files', - '[d]: Get descriptions', - '[f]: Get files', - '[p] Print search results', - '[q] Quit', sep='\n') - elif code == 'q': - print('Bye.', color='alt') - return - elif code == 'd': - print_descriptions(choices, mags, site, identifiers) - elif code == 'f': - print_file_lists(choices, mags, site, identifiers) - elif code == 'p': - print_search_results(mags, sizes, uploaded) - elif code == 'm': - save_magnets(choices, mags, - config.get('Save', 'directory')) - elif code == 't': - save_torrents(choices, mags, - config.get('Save', 'directory')) - elif not l: - print('No links entered!', color='WARN') - else: - break - except Exception as e: - print('Exception:', e, color='ERROR') - choices = () - - save_to_file = False - - if args.save_magnets or config.getboolean('Save', 'magnets'): - print('Saving selected magnets...') - save_magnets(choices, mags, config.get('Save', 'directory')) - save_to_file = True - - if args.save_torrents or config.getboolean('Save', 'torrents'): - print('Saving selected torrents...') - save_torrents(choices, mags, config.get('Save', 'directory')) - save_to_file = True - - if save_to_file: - return - - for choice in choices: - url = mags[int(choice)][0] - - if args.transmission or config.getboolean('Misc', 'transmission'): - subprocess.call(transmission_command + ['-l', '--add', url], shell=False) - subprocess.call(transmission_command + ['-l']) - - elif args.command or config.get('Misc', 'openCommand'): - command = config.get('Misc', 'openCommand') - if args.command: - command = args.command - subprocess.call(parse_cmd(command, url), shell=False) - - else: - webbrowser.open(url) - - -if __name__ == '__main__': - main() diff --git a/pirate/__init__.py b/pirate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pirate/data.py b/pirate/data.py new file mode 100644 index 0000000..1076a64 --- /dev/null +++ b/pirate/data.py @@ -0,0 +1,8 @@ +import json + +categories = json.load(open('data/categories.json')) +sorts = json.load(open('data/sorts.json')) + +default_headers = {'User-Agent': 'pirate get'} +default_timeout = 10 +colored_output = True \ No newline at end of file diff --git a/pirate/local.py b/pirate/local.py new file mode 100644 index 0000000..2c2c3f2 --- /dev/null +++ b/pirate/local.py @@ -0,0 +1,41 @@ +import urllib.parse as parse +import html.parser as parser + +# create a subclass and override the handler methods +class BayParser(parser.HTMLParser): + title = '' + q = '' + state = 'looking' + results = [] + + def __init__(self, q): + super().__init__(self) + self.q = q.lower() + + def handle_starttag(self, tag, attrs): + if tag == 'title': + self.state = 'title' + if tag == 'magnet' and self.state == 'matched': + self.state = 'magnet' + + def handle_data(self, data): + if self.state == 'title': + if data.lower().find(self.q) != -1: + self.title = data + self.state = 'matched' + else: + self.state = 'looking' + if self.state == 'magnet': + self.results.append([ + 'magnet:?xt=urn:btih:' + + parse.quote(data) + + '&dn=' + + parse.quote(self.title), '?', '?']) + self.state = 'looking' + + +def search(db, terms): + xml = open(db).readlines() + parser = BayParser(' '.join(terms)) + parser.feed(''.join(xml)) + return parser.results \ No newline at end of file diff --git a/pirate/pirate.py b/pirate/pirate.py new file mode 100755 index 0000000..e3ae9dc --- /dev/null +++ b/pirate/pirate.py @@ -0,0 +1,308 @@ +import re +import os +import argparse +import subprocess +import configparser +import socket +import urllib.request as request +import urllib.error +import webbrowser + +import pirate.data +import pirate.torrent +import pirate.local +import pirate.print + +from os.path import expanduser, expandvars +from pirate.print import print + + +def load_config(): + config = configparser.ConfigParser() + + # default options + config.add_section('Save') + config.set('Save', 'magnets', 'false') + config.set('Save', 'torrents', 'false') + config.set('Save', 'directory', os.getcwd()) + + config.add_section('LocalDB') + config.set('LocalDB', 'enabled', 'false') + config.set('LocalDB', 'path', expanduser('~/downloads/pirate-get/db')) + + config.add_section('Misc') + config.set('Misc', 'openCommand', '') + config.set('Misc', 'transmission', 'false') + config.set('Misc', 'colors', 'true') + + # user-defined config files + main = expandvars('$XDG_CONFIG_HOME/pirate-get') + alt = expanduser('~/.config/pirate-get') + + # read config file + config.read([main] if os.path.isfile(main) else [alt]) + + # expand env variables + directory = expanduser(expandvars(config.get('Save', 'Directory'))) + path = expanduser(expandvars(config.get('LocalDB', 'path'))) + + config.set('Save', 'Directory', directory) + config.set('LocalDB', 'path', path) + + return config + + +def parse_cmd(cmd, url): + cmd_args_regex = r'''(('[^']*'|"[^"]*"|(\\\s|[^\s])+)+ *)''' + ret = re.findall(cmd_args_regex, cmd) + ret = [i[0].strip().replace('%s', url) for i in ret] + ret_no_quotes = [] + for item in ret: + if ((item[0] == "'" and item[-1] == "'") or ( + item[0] == '"' and item[-1] == '"')): + ret_no_quotes.append(item[1:-1]) + else: + ret_no_quotes.append(item) + return ret_no_quotes + + +def main(): + config = load_config() + + parser = argparse.ArgumentParser( + description='finds and downloads torrents from the Pirate Bay') + parser.add_argument('-b', dest='browse', + action='store_true', + help='display in Browse mode') + parser.add_argument('search', metavar='search', + nargs='*', help='term to search for') + parser.add_argument('-c', dest='category', metavar='category', + help='specify a category to search', default='All') + parser.add_argument('-s', dest='sort', metavar='sort', + help='specify a sort option', default='SeedersDsc') + parser.add_argument('-R', dest='recent', action='store_true', + help='torrents uploaded in the last 48hours.' + '*ignored in searches*') + parser.add_argument('-l', dest='list_categories', + action='store_true', + help='list categories') + parser.add_argument('--list_sorts', dest='list_sorts', + action='store_true', + help='list Sortable Types') + parser.add_argument('-L', '--local', dest='database', + help='an xml file containing the Pirate Bay database') + parser.add_argument('-p', dest='pages', default=1, + help='the number of pages to fetch ' + "(doesn't work with --local)") + parser.add_argument('-0', dest='first', + action='store_true', + help='choose the top result') + parser.add_argument('-a', '--download-all', + action='store_true', + help='download all results') + parser.add_argument('-t', '--transmission', + action='store_true', + help='open magnets with transmission-remote') + parser.add_argument('-P', '--port', dest='port', + help='transmission-remote rpc port. default is 9091') + parser.add_argument('-C', '--custom', dest='command', + help='open magnets with a custom command' + ' (%%s will be replaced with the url)') + parser.add_argument('-M', '--save-magnets', + action='store_true', + help='save magnets links as files') + parser.add_argument('-T', '--save-torrents', + action='store_true', + help='save torrent files') + parser.add_argument('-S', '--save-directory', + type=str, metavar='DIRECTORY', + help='directory where to save downloaded files' + ' (if none is given $PWD will be used)') + parser.add_argument('--disable-colors', dest='color', + action='store_false', + help='disable colored output') + args = parser.parse_args() + + if (config.getboolean('Misc', 'colors') and not args.color + or not config.getboolean('Misc', 'colors')): + #global colored_output + pirate.data.colored_output = False + + if args.save_directory: + config.set('Save', 'directory', args.save_directory) + + transmission_command = ['transmission-remote'] + if args.port: + transmission_command.append(args.port) + + if args.transmission or config.getboolean('Misc', 'transmission'): + ret = subprocess.call(transmission_command + ['-l'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if ret != 0: + print('Transmission is not running.') + return + + if args.list_categories: + cur_color = 'zebra_0' + for key, value in sorted(pirate.data.categories.items()): + cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' + print(str(value), '\t', key, sep='', color=cur_color) + return + + if args.list_sorts: + cur_color = 'zebra_0' + for key, value in sorted(sorts.items()): + cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' + print(str(value), '\t', key, sep='', color=cur_color) + return + + if args.database or config.getboolean('LocalDB', 'enabled'): + if args.database: + path = args.database + else: + path = config.get('LocalDB', 'path') + mags = pirate.local.search(path, args.search) + sizes, uploaded = [], [] + + else: + mags, mirrors = [], {'https://thepiratebay.se'} + try: + req = request.Request('https://proxybay.co/list.txt', + headers=pirate.data.default_headers) + f = request.urlopen(req, timeout=pirate.data.default_timeout) + except IOError: + print('Could not fetch additional mirrors', color='WARN') + else: + if f.getcode() != 200: + raise IOError('The proxy bay responded with an error.') + mirrors = mirrors.union([i.decode('utf-8').strip() + for i in f.readlines()][3:]) + + for mirror in mirrors: + try: + print('Trying', mirror, end='... ') + mags, sizes, uploaded, ids = pirate.torrent.remote(args, mirror) + except (urllib.error.URLError, socket.timeout, + IOError, ValueError): + print('Failed', color='WARN') + else: + site = mirror + print('Ok', color='alt') + break + else: + print('No available mirrors :(', color='WARN') + return + + if not mags: + print('No results') + return + + pirate.print.search_results(mags, sizes, uploaded, local=args.database) + + if args.first: + print('Choosing first result') + choices = [0] + elif args.download_all: + print('Downloading all results') + choices = range(len(mags)) + else: + # New input loop to support different link options + while True: + print("\nSelect links (Type 'h' for more options" + ", 'q' to quit)", end='\b', color='alt') + try: + l=input(': ') + except KeyboardInterrupt : + print('\nCancelled.') + return + + try: + # Very permissive handling + # Check for any occurances or d, f, p, t, m, or q + cmd_code_match = re.search(r'([hdfpmtq])', l, + flags=re.IGNORECASE) + if cmd_code_match: + code = cmd_code_match.group(0).lower() + else: + code = None + + # Clean up command codes + # Substitute multiple consecutive spaces/commas for single + # comma remove anything that isn't an integer or comma. + # Turn into list + l = re.sub(r'^[hdfp, ]*|[hdfp, ]*$', '', l) + l = re.sub('[ ,]+', ',', l) + l = re.sub('[^0-9,]', '', l) + choices = l.split(',') + + # Act on option, if supplied + print('') + if code == 'h': + print('Options:', + ': Download selected torrents', + '[m]: Save magnets as files', + '[t]: Save .torrent files', + '[d]: Get descriptions', + '[f]: Get files', + '[p] Print search results', + '[q] Quit', sep='\n') + elif code == 'q': + print('Bye.', color='alt') + return + elif code == 'd': + pirate.print.descriptions(choices, mags, site, ids) + elif code == 'f': + pirate.print.file_lists(choices, mags, site, ids) + elif code == 'p': + pirate.print.search_results(mags, sizes, uploaded) + elif code == 'm': + save_magnets(choices, mags, + config.get('Save', 'directory')) + elif code == 't': + save_torrents(choices, mags, + config.get('Save', 'directory')) + elif not l: + print('No links entered!', color='WARN') + else: + break + except Exception as e: + print('Exception:', e, color='ERROR') + choices = () + + save_to_file = False + + if args.save_magnets or config.getboolean('Save', 'magnets'): + print('Saving selected magnets...') + pirate.torrent.save_magnets(choices, mags, + config.get('Save', 'directory')) + save_to_file = True + + if args.save_torrents or config.getboolean('Save', 'torrents'): + print('Saving selected torrents...') + pirate.torrent.save_torrents(choices, mags, + config.get('Save', 'directory')) + save_to_file = True + + if save_to_file: + return + + for choice in choices: + url = mags[int(choice)][0] + + if args.transmission or config.getboolean('Misc', 'transmission'): + subprocess.call(transmission_command + ['-l', '--add', url]) + subprocess.call(transmission_command + ['-l']) + + elif args.command or config.get('Misc', 'openCommand'): + command = config.get('Misc', 'openCommand') + if args.command: + command = args.command + subprocess.call(parse_cmd(command, url)) + + else: + webbrowser.open(url) + + +if __name__ == '__main__': + main() diff --git a/pirate/print.py b/pirate/print.py new file mode 100644 index 0000000..7e0f612 --- /dev/null +++ b/pirate/print.py @@ -0,0 +1,126 @@ +import builtins +import re +import os +import gzip +import colorama +import urllib.parse as parse +import urllib.request as request +from io import BytesIO + +import pirate.data + + +def print(*args, **kwargs): + if kwargs.get('color', False) and pirate.data.colored_output: + colorama.init() + color_dict = { + 'default': '', + 'header': colorama.Back.BLACK + colorama.Fore.WHITE, + 'alt': colorama.Fore.YELLOW, + 'zebra_0': '', + 'zebra_1': colorama.Fore.BLUE, + 'WARN': colorama.Fore.MAGENTA, + 'ERROR': colorama.Fore.RED} + + c = color_dict[kwargs.pop('color')] + args = (c + args[0],) + args[1:] + (colorama.Style.RESET_ALL,) + kwargs.pop('color', None) + return builtins.print(*args, **kwargs) + else: + kwargs.pop('color', None) + return builtins.print(*args, **kwargs) + + +def search_results(mags, sizes, uploaded, local): + columns = int(os.popen('stty size', 'r').read().split()[1]) + cur_color = 'zebra_0' + + if local: + print('{:>4} {:{length}}'.format( + 'LINK', 'NAME', length=columns - 8), + color='header') + else: + print('{:>4} {:>5} {:>5} {:>5} {:9} {:11} {:{length}}'.format( + 'LINK', 'SEED', 'LEECH', 'RATIO', + 'SIZE', 'UPLOAD', 'NAME', length=columns - 52), + color='header') + + for m, magnet in enumerate(mags): + # Alternate between colors + cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' + + name = re.search(r'dn=([^\&]*)', magnet[0]) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + + if local: + line = '{:5} {:{length}}' + content = [m, torrent_name[:columns]] + else: + no_seeders, no_leechers = map(int, magnet[1:]) + size, unit = (float(sizes[m][0]), + sizes[m][1]) if sizes else (0, '???') + date = uploaded[m] + + # compute the S/L ratio (Higher is better) + try: + ratio = no_seeders / no_leechers + except ZeroDivisionError: + ratio = float('inf') + + line = ('{:4} {:5} {:5} {:5.1f} {:5.1f}' + ' {:3} {:<11} {:{length}}') + content = [m, no_seeders, no_leechers, ratio, + size, unit, date, torrent_name[:columns - 52]] + + # enhanced print output with justified columns + print(line.format(*content, length=columns - 52), color=cur_color) + + +def descriptions(chosen_links, mags, site, identifiers): + for link in chosen_links: + link = int(link) + path = '/torrent/%s/' % identifiers[link] + req = request.Request(site + path, headers=default_headers) + req.add_header('Accept-encoding', 'gzip') + f = request.urlopen(req, timeout=default_timeout) + + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + + res = f.read().decode('utf-8') + name = re.search(r'dn=([^\&]*)', mags[link][0]) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + desc = re.search(r'
\s*
(.+?)(?=
)', + res, re.DOTALL).group(1) + + # Replace HTML links with markdown style versions + desc = re.sub(r']*>(\s*)([^<]+?)(\s*' + r')', r'\2[\3](\1)\4', desc) + + print('Description for "%s":' % torrent_name, color='zebra_1') + print(desc, color='zebra_0') + + +def file_lists(chosen_links, mags, site, identifiers): + for link in chosen_links: + path = '/ajax_details_filelist.php' + query = '?id=' + identifiers[int(link)] + req = request.Request(site + path + query, headers=default_headers) + req.add_header('Accept-encoding', 'gzip') + f = request.urlopen(req, timeout=default_timeout) + + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + + res = f.read().decode('utf-8').replace(' ', ' ') + files = re.findall(r'\s*([^<]+?)\s*\s*([^<]+?)\s*', res) + name = re.search(r'dn=([^\&]*)', mags[int(link)][0]) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + + print('Files in "%s":' % torrent_name, color='zebra_1') + cur_color = 'zebra_0' + + for f in files: + print('{0[0]:>11} {0[1]}'.format(f), color=cur_color) + cur_color = 'zebra_0' if (cur_color == 'zebra_1') else 'zebra_1' \ No newline at end of file diff --git a/pirate/torrent.py b/pirate/torrent.py new file mode 100644 index 0000000..bdaff2d --- /dev/null +++ b/pirate/torrent.py @@ -0,0 +1,155 @@ +import re +import sys +import gzip +import urllib.request as request +import urllib.parse as parse +import urllib.error + +import pirate.data + +from io import BytesIO + +#todo: redo this with html parser instead of regex +def remote(args, mirror): + res_l = [] + pages = int(args.pages) + if pages < 1: + raise ValueError('Please provide an integer greater than 0 ' + 'for the number of pages to fetch.') + + if str(args.category) in pirate.data.categories.values(): + category = args.category + elif args.category in pirate.data.categories.keys(): + category = pirate.data.categories[args.category] + else: + category = '0' + print('Invalid category ignored', color='WARN') + + if str(args.sort) in pirate.data.sorts.values(): + sort = args.sort + elif args.sort in pirate.data.sorts.keys(): + sort = pirate.data.sorts[args.sort] + else: + sort = '99' + print('Invalid sort ignored', color='WARN') + # Catch the Ctrl-C exception and exit cleanly + try: + sizes = [] + uploaded = [] + identifiers = [] + for page in range(pages): + if args.browse: + path = '/browse/' + if(category == 0): + category = 100 + path = '/browse/' + '/'.join(str(i) for i in ( + category, page, sort)) + elif len(args.search) == 0: + path = '/top/48h' if args.recent else '/top/' + if(category == 0): + path += 'all' + else: + path += str(category) + else: + path = '/search/' + '/'.join(str(i) for i in ( + '+'.join(args.search), + page, sort, + category)) + + req = request.Request(mirror + path, + headers=pirate.data.default_headers) + req.add_header('Accept-encoding', 'gzip') + f = request.urlopen(req, timeout=pirate.data.default_timeout) + if f.info().get('Content-Encoding') == 'gzip': + f = gzip.GzipFile(fileobj=BytesIO(f.read())) + res = f.read().decode('utf-8') + found = re.findall(r'"(magnet\:\?xt=[^"]*)|' + r'([^<]+)', res) + + # check for a blocked mirror + no_results = re.search(r'No hits\. Try adding an asterisk in ' + r'you search phrase\.', res) + if found == [] and no_results is None: + # Contradiction - we found no results, + # but the page didn't say there were no results. + # The page is probably not actually the pirate bay, + # so let's try another mirror + raise IOError('Blocked mirror detected.') + + # get sizes as well and substitute the   character + sizes.extend([match.replace(' ', ' ').split() + for match in re.findall(r'(?<=Size )[0-9.]' + r'+\ \;[KMGT]*[i ]*B', res)]) + + uploaded.extend([match.replace(' ', ' ') + for match in re.findall(r'(?<=Uploaded )' + r'.+(?=\, Size)',res)]) + + identifiers.extend([match.replace(' ', ' ') + for match in re.findall('(?<=/torrent/)' + '[0-9]+(?=/)',res)]) + + state = 'seeds' + curr = ['', 0, 0] #magnet, seeds, leeches + for f in found: + if f[1] == '': + curr[0] = f[0] + else: + if state == 'seeds': + curr[1] = f[1] + state = 'leeches' + else: + curr[2] = f[1] + state = 'seeds' + res_l.append(curr) + curr = ['', 0, 0] + except KeyboardInterrupt : + print('\nCancelled.') + sys.exit(0) + + # return the sizes in a spearate list + return res_l, sizes, uploaded, identifiers + + + +def get_torrent(info_hash): + url = 'http://torcache.net/torrent/{:X}.torrent' + req = request.Request(url.format(info_hash), + headers=pirate.data.default_headers) + req.add_header('Accept-encoding', 'gzip') + + torrent = request.urlopen(req, timeout=pirate.data.default_timeout) + if torrent.info().get('Content-Encoding') == 'gzip': + torrent = gzip.GzipFile(fileobj=BytesIO(torrent.read())) + + return torrent.read() + + +def save_torrents(chosen_links, mags, folder): + for link in chosen_links: + magnet = mags[int(link)][0] + name = re.search(r'dn=([^\&]*)', magnet) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) + file = os.path.join(folder, torrent_name + '.torrent') + + try: + torrent = get_torrent(info_hash) + except urllib.error.HTTPError: + print('There is no cached file for this torrent :(', color='ERROR') + else: + open(file,'wb').write(torrent) + print('Saved {:X} in {}'.format(info_hash, file)) + + +def save_magnets(chosen_links, mags, folder): + for link in chosen_links: + magnet = mags[int(link)][0] + name = re.search(r'dn=([^\&]*)', magnet) + torrent_name = parse.unquote(name.group(1)).replace('+', ' ') + info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) + file = os.path.join(folder, torrent_name + '.magnet') + + print('Saved {:X} in {}'.format(info_hash, file)) + with open(file, 'w') as f: + f.write(magnet + '\n') \ No newline at end of file diff --git a/src/pirate-get.py b/src/pirate-get.py deleted file mode 100755 index 4a9aeaa..0000000 --- a/src/pirate-get.py +++ /dev/null @@ -1,696 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2015, Viktor Stanchev and contributors -# -# This file is part of pirate-get. -# -# pirate-get is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# pirate-get is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with pirate-get. If not, see . - -import os -import sys -import re -import string -import gzip -import configparser -import argparse -import builtins -import subprocess - -import webbrowser -import urllib.request as request -import urllib.parse as parse - -from html.parser import HTMLParser -from urllib.error import URLError, HTTPError -from socket import timeout -from io import BytesIO -from os.path import expanduser, expandvars - -colored_output = True - -default_timeout = 10 - -default_headers = {'User-Agent': 'pirate get'} - -categories = { - 'All': 0, - 'Applications': 300, - 'Applications/Android': 306, - 'Applications/Handheld': 304, - 'Applications/IOS (iPad/iPhone)': 305, - 'Applications/Mac': 302, - 'Applications/Other OS': 399, - 'Applications/UNIX': 303, - 'Applications/Windows': 301, - 'Audio': 100, - 'Audio/Audio books': 102, - 'Audio/FLAC': 104, - 'Audio/Music': 101, - 'Audio/Other': 199, - 'Audio/Sound clips': 103, - 'Games': 400, - 'Games/Android': 408, - 'Games/Handheld': 406, - 'Games/IOS (iPad/iPhone)': 407, - 'Games/Mac': 402, - 'Games/Other': 499, - 'Games/PC': 401, - 'Games/PSx': 403, - 'Games/Wii': 405, - 'Games/XBOX360': 404, - 'Other': 600, - 'Other/Comics': 602, - 'Other/Covers': 604, - 'Other/E-books': 601, - 'Other/Other': 699, - 'Other/Physibles': 605, - 'Other/Pictures': 603, - 'Porn': 500, - 'Porn/Games': 504, - 'Porn/HD - Movies': 505, - 'Porn/Movie clips': 506, - 'Porn/Movies': 501, - 'Porn/Movies DVDR': 502, - 'Porn/Other': 599, - 'Porn/Pictures': 503, - 'Video': 200, - 'Video/3D': 209, - 'Video/HD - Movies': 207, - 'Video/HD - TV shows': 208, - 'Video/Handheld': 206, - 'Video/Movie clips': 204, - 'Video/Movies': 201, - 'Video/Movies DVDR': 202, - 'Video/Music videos': 203, - 'Video/Other': 299, - 'Video/TV shows': 205} - -sorts = { - 'TitleDsc': 1, 'TitleAsc': 2, - 'DateDsc': 3, 'DateAsc': 4, - 'SizeDsc': 5, 'SizeAsc': 6, - 'SeedersDsc': 7, 'SeedersAsc': 8, - 'LeechersDsc': 9, 'LeechersAsc': 10, - 'CategoryDsc': 13, 'CategoryAsc': 14, - 'Default': 99} - - -# create a subclass and override the handler methods -class BayParser(HTMLParser): - title = '' - q = '' - state = 'looking' - results = [] - - def __init__(self, q): - HTMLParser.__init__(self) - self.q = q.lower() - - def handle_starttag(self, tag, attrs): - if tag == 'title': - self.state = 'title' - if tag == 'magnet' and self.state == 'matched': - self.state = 'magnet' - - def handle_data(self, data): - if self.state == 'title': - if data.lower().find(self.q) != -1: - self.title = data - self.state = 'matched' - else: - self.state = 'looking' - if self.state == 'magnet': - self.results.append([ - 'magnet:?xt=urn:btih:' + - parse.quote(data) + - '&dn=' + - parse.quote(self.title), '?', '?']) - self.state = 'looking' - - -def print(*args, **kwargs): - if kwargs.get('color', False) and colored_output: - try: - import colorama - except (ImportError): - pass - else: - colorama.init() - color_dict = { - 'default': '', - 'header': colorama.Back.BLACK + colorama.Fore.WHITE, - 'alt': colorama.Fore.YELLOW, - 'zebra_0': '', - 'zebra_1': colorama.Fore.BLUE, - 'WARN': colorama.Fore.MAGENTA, - 'ERROR': colorama.Fore.RED} - - c = color_dict[kwargs.pop('color')] - args = (c + args[0],) + args[1:] + (colorama.Style.RESET_ALL,) - kwargs.pop('color', None) - return builtins.print(*args, **kwargs) - else: - kwargs.pop('color', None) - return builtins.print(*args, **kwargs) - - -def parse_cmd(cmd, url): - cmd_args_regex = r'''(('[^']*'|"[^"]*"|(\\\s|[^\s])+)+ *)''' - ret = re.findall(cmd_args_regex, cmd) - ret = [i[0].strip().replace('%s', url) for i in ret] - ret_no_quotes = [] - for item in ret: - if (item[0] == "'" and item[-1] == "'") or (item[0] == '"' and item[-1] == '"'): - ret_no_quotes.append(item[1:-1]) - else: - ret_no_quotes.append(item) - return ret_no_quotes - - -#todo: redo this with html parser instead of regex -def remote(args, mirror): - res_l = [] - pages = int(args.pages) - if pages < 1: - raise ValueError('Please provide an integer greater than 0 ' - 'for the number of pages to fetch.') - - if str(args.category) in categories.values(): - category = args.category - elif args.category in categories.keys(): - category = categories[args.category] - else: - category = '0' - print('Invalid category ignored', color='WARN') - - if str(args.sort) in sorts.values(): - sort = args.sort - elif args.sort in sorts.keys(): - sort = sorts[args.sort] - else: - sort = '99' - print('Invalid sort ignored', color='WARN') - # Catch the Ctrl-C exception and exit cleanly - try: - sizes = [] - uploaded = [] - identifiers = [] - for page in range(pages): - if args.browse: - path = '/browse/' - if(category == 0): - category = 100 - path = '/browse/' + '/'.join(str(i) for i in ( - category, page, sort)) - elif len(args.search) == 0: - path = '/top/48h' if args.recent else '/top/' - if(category == 0): - path += 'all' - else: - path += str(category) - else: - path = '/search/' + '/'.join(str(i) for i in ( - '+'.join(args.search), - page, sort, - category)) - - req = request.Request(mirror + path, headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - res = f.read().decode('utf-8') - found = re.findall(r'"(magnet\:\?xt=[^"]*)|' - r'([^<]+)', res) - - # check for a blocked mirror - no_results = re.search(r'No hits\. Try adding an asterisk in ' - r'you search phrase\.', res) - if found == [] and no_results is None: - # Contradiction - we found no results, - # but the page didn't say there were no results. - # The page is probably not actually the pirate bay, - # so let's try another mirror - raise IOError('Blocked mirror detected.') - - # get sizes as well and substitute the   character - sizes.extend([match.replace(' ', ' ').split() - for match in re.findall(r'(?<=Size )[0-9.]' - r'+\ \;[KMGT]*[i ]*B', res)]) - - uploaded.extend([match.replace(' ', ' ') - for match in re.findall(r'(?<=Uploaded )' - r'.+(?=\, Size)',res)]) - - identifiers.extend([match.replace(' ', ' ') - for match in re.findall('(?<=/torrent/)' - '[0-9]+(?=/)',res)]) - - state = 'seeds' - curr = ['', 0, 0] #magnet, seeds, leeches - for f in found: - if f[1] == '': - curr[0] = f[0] - else: - if state == 'seeds': - curr[1] = f[1] - state = 'leeches' - else: - curr[2] = f[1] - state = 'seeds' - res_l.append(curr) - curr = ['', 0, 0] - except KeyboardInterrupt : - print('\nCancelled.') - sys.exit(0) - - # return the sizes in a spearate list - return res_l, sizes, uploaded, identifiers - - -def local(db, search): - xml = open(db).readlines() - parser = BayParser(' '.join(search)) - parser.feed(''.join(xml)) - return parser.results - - -def load_config(): - config = configparser.ConfigParser() - - # default options - config.add_section('Save') - config.set('Save', 'magnets', 'false') - config.set('Save', 'torrents', 'false') - config.set('Save', 'directory', os.getcwd()) - - config.add_section('LocalDB') - config.set('LocalDB', 'enabled', 'false') - config.set('LocalDB', 'path', expanduser('~/downloads/pirate-get/db')) - - config.add_section('Misc') - config.set('Misc', 'openCommand', '') - config.set('Misc', 'transmission', 'false') - config.set('Misc', 'colors', 'true') - - # user-defined config files - main = expandvars('$XDG_CONFIG_HOME/pirate-get') - alt = expanduser('~/.config/pirate-get') - - # read config file - config.read([main] if os.path.isfile(main) else [alt]) - - # expand env variables - directory = expanduser(expandvars(config.get('Save', 'Directory'))) - path = expanduser(expandvars(config.get('LocalDB', 'path'))) - - config.set('Save', 'Directory', directory) - config.set('LocalDB', 'path', path) - - return config - - -def get_torrent(info_hash): - url = 'http://torcache.net/torrent/{:X}.torrent' - req = request.Request(url.format(info_hash), headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - - torrent = request.urlopen(req, timeout=default_timeout) - if torrent.info().get('Content-Encoding') == 'gzip': - torrent = gzip.GzipFile(fileobj=BytesIO(torrent.read())) - - return torrent.read() - - -def print_search_results(mags, sizes, uploaded, local): - columns = int(os.popen('stty size', 'r').read().split()[1]) - cur_color = 'zebra_0' - - if local: - print('{:>4} {:{length}}'.format( - 'LINK', 'NAME', length=columns - 8), - color='header') - else: - print('{:>4} {:>5} {:>5} {:>5} {:9} {:11} {:{length}}'.format( - 'LINK', 'SEED', 'LEECH', 'RATIO', - 'SIZE', 'UPLOAD', 'NAME', length=columns - 52), - color='header') - - for m, magnet in enumerate(mags): - # Alternate between colors - cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' - - name = re.search(r'dn=([^\&]*)', magnet[0]) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - - if local: - line = '{:5} {:{length}}' - content = [m, torrent_name[:columns]] - else: - no_seeders, no_leechers = map(int, magnet[1:]) - size, unit = (float(sizes[m][0]), sizes[m][1]) if sizes else (0, '???') - date = uploaded[m] - - # compute the S/L ratio (Higher is better) - try: - ratio = no_seeders / no_leechers - except ZeroDivisionError: - ratio = float('inf') - - line = ('{:4} {:5} {:5} {:5.1f} {:5.1f}' - ' {:3} {:<11} {:{length}}') - content = [m, no_seeders, no_leechers, ratio, - size, unit, date, torrent_name[:columns - 52]] - - # enhanced print output with justified columns - print(line.format(*content, length=columns - 52), color=cur_color) - - -def print_descriptions(chosen_links, mags, site, identifiers): - for link in chosen_links: - link = int(link) - path = '/torrent/%s/' % identifiers[link] - req = request.Request(site + path, headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) - - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - - res = f.read().decode('utf-8') - name = re.search(r'dn=([^\&]*)', mags[link][0]) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - desc = re.search(r'
\s*
(.+?)(?=
)', - res, re.DOTALL).group(1) - - # Replace HTML links with markdown style versions - desc = re.sub(r']*>(\s*)([^<]+?)(\s*' - r')', r'\2[\3](\1)\4', desc) - - print('Description for "%s":' % torrent_name, color='zebra_1') - print(desc, color='zebra_0') - - -def print_file_lists(chosen_links, mags, site, identifiers): - for link in chosen_links: - path = '/ajax_details_filelist.php' - query = '?id=' + identifiers[int(link)] - req = request.Request(site + path + query, headers=default_headers) - req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) - - if f.info().get('Content-Encoding') == 'gzip': - f = gzip.GzipFile(fileobj=BytesIO(f.read())) - - res = f.read().decode('utf-8').replace(' ', ' ') - files = re.findall(r'\s*([^<]+?)\s*\s*([^<]+?)\s*', res) - name = re.search(r'dn=([^\&]*)', mags[int(link)][0]) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - - print('Files in "%s":' % torrent_name, color='zebra_1') - cur_color = 'zebra_0' - - for f in files: - print('{0[0]:>11} {0[1]}'.format(f), color=cur_color) - cur_color = 'zebra_0' if (cur_color == 'zebra_1') else 'zebra_1' - - -def save_torrents(chosen_links, mags, folder): - for link in chosen_links: - magnet = mags[int(link)][0] - name = re.search(r'dn=([^\&]*)', magnet) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) - file = os.path.join(folder, torrent_name + '.torrent') - - try: - torrent = get_torrent(info_hash) - except HTTPError: - print('There is no cached file for this torrent :(', color='ERROR') - else: - open(file,'wb').write(torrent) - print('Saved {:X} in {}'.format(info_hash, file)) - - -def save_magnets(chosen_links, mags, folder): - for link in chosen_links: - magnet = mags[int(link)][0] - name = re.search(r'dn=([^\&]*)', magnet) - torrent_name = parse.unquote(name.group(1)).replace('+', ' ') - info_hash = int(re.search(r'btih:([a-f0-9]{40})', magnet).group(1), 16) - file = os.path.join(folder, torrent_name + '.magnet') - - print('Saved {:X} in {}'.format(info_hash, file)) - with open(file, 'w') as f: - f.write(magnet + '\n') - - -def main(): - config = load_config() - - parser = argparse.ArgumentParser( - description='finds and downloads torrents from the Pirate Bay') - parser.add_argument('-b', dest='browse', - action='store_true', - help='display in Browse mode') - parser.add_argument('search', metavar='search', - nargs='*', help='term to search for') - parser.add_argument('-c', dest='category', metavar='category', - help='specify a category to search', default='All') - parser.add_argument('-s', dest='sort', metavar='sort', - help='specify a sort option', default='SeedersDsc') - parser.add_argument('-R', dest='recent', action='store_true', - help='torrents uploaded in the last 48hours.' - '*ignored in searches*') - parser.add_argument('-l', dest='list_categories', - action='store_true', - help='list categories') - parser.add_argument('--list_sorts', dest='list_sorts', - action='store_true', - help='list Sortable Types') - parser.add_argument('-L', '--local', dest='database', - help='an xml file containing the Pirate Bay database') - parser.add_argument('-p', dest='pages', default=1, - help='the number of pages to fetch ' - "(doesn't work with --local)") - parser.add_argument('-0', dest='first', - action='store_true', - help='choose the top result') - parser.add_argument('-a', '--download-all', - action='store_true', - help='download all results') - parser.add_argument('-t', '--transmission', - action='store_true', - help='open magnets with transmission-remote') - parser.add_argument('-P', '--port', dest='port', - help='transmission-remote rpc port. default is 9091') - parser.add_argument('-C', '--custom', dest='command', - help='open magnets with a custom command' - ' (%%s will be replaced with the url)') - parser.add_argument('-M', '--save-magnets', - action='store_true', - help='save magnets links as files') - parser.add_argument('-T', '--save-torrents', - action='store_true', - help='save torrent files') - parser.add_argument('-S', '--save-directory', - type=str, metavar='DIRECTORY', - help='directory where to save downloaded files' - ' (if none is given $PWD will be used)') - parser.add_argument('--disable-colors', dest='color', - action='store_false', - help='disable colored output') - args = parser.parse_args() - - if (config.getboolean('Misc', 'colors') and not args.color - or not config.getboolean('Misc', 'colors')): - global colored_output - colored_output = False - - if args.save_directory: - config.set('Save', 'directory', args.save_directory) - - transmission_command = ['transmission-remote'] - if args.port: - transmission_command.append(args.port) - - if args.transmission or config.getboolean('Misc', 'transmission'): - ret = subprocess.call(transmission_command + ['-l'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - if ret != 0: - print('Transmission is not running.') - return - - if args.list_categories: - cur_color = 'zebra_0' - for key, value in sorted(categories.items()) : - cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' - print(str(value), '\t', key, sep='', color=cur_color) - return - - if args.list_sorts: - cur_color = 'zebra_0' - for key, value in sorted(sorts.items()): - cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' - print(str(value), '\t', key, sep='', color=cur_color) - return - - if args.database or config.getboolean('LocalDB', 'enabled'): - if args.database: - path = args.database - else: - path = config.get('LocalDB', 'path') - mags = local(path, args.search) - sizes, uploaded = [], [] - - else: - mags, mirrors = [], {'https://thepiratebay.se'} - try: - req = request.Request('https://proxybay.co/list.txt', - headers=default_headers) - f = request.urlopen(req, timeout=default_timeout) - except IOError: - print('Could not fetch additional mirrors', color='WARN') - else: - if f.getcode() != 200: - raise IOError('The proxy bay responded with an error.') - mirrors = mirrors.union([i.decode('utf-8').strip() - for i in f.readlines()][3:]) - - for mirror in mirrors: - try: - print('Trying', mirror, end='... ') - mags, sizes, uploaded, identifiers = remote(args, mirror) - except (URLError, IOError, ValueError, timeout): - print('Failed', color='WARN') - else: - site = mirror - print('Ok', color='alt') - break - else: - print('No available mirrors :(', color='WARN') - return - - if not mags: - print('No results') - return - - print_search_results(mags, sizes, uploaded, local=args.database) - - if args.first: - print('Choosing first result') - choices = [0] - elif args.download_all: - print('Downloading all results') - choices = range(len(mags)) - else: - # New input loop to support different link options - while True: - print("\nSelect links (Type 'h' for more options" - ", 'q' to quit)", end='\b', color='alt') - try: - l=input(': ') - except KeyboardInterrupt : - print('\nCancelled.') - return - - try: - # Very permissive handling - # Check for any occurances or d, f, p, t, m, or q - cmd_code_match = re.search(r'([hdfpmtq])', l, - flags=re.IGNORECASE) - if cmd_code_match: - code = cmd_code_match.group(0).lower() - else: - code = None - - # Clean up command codes - # Substitute multiple consecutive spaces/commas for single - # comma remove anything that isn't an integer or comma. - # Turn into list - l = re.sub(r'^[hdfp, ]*|[hdfp, ]*$', '', l) - l = re.sub('[ ,]+', ',', l) - l = re.sub('[^0-9,]', '', l) - choices = l.split(',') - - # Act on option, if supplied - print('') - if code == 'h': - print('Options:', - ': Download selected torrents', - '[m]: Save magnets as files', - '[t]: Save .torrent files', - '[d]: Get descriptions', - '[f]: Get files', - '[p] Print search results', - '[q] Quit', sep='\n') - elif code == 'q': - print('Bye.', color='alt') - return - elif code == 'd': - print_descriptions(choices, mags, site, identifiers) - elif code == 'f': - print_file_lists(choices, mags, site, identifiers) - elif code == 'p': - print_search_results(mags, sizes, uploaded) - elif code == 'm': - save_magnets(choices, mags, - config.get('Save', 'directory')) - elif code == 't': - save_torrents(choices, mags, - config.get('Save', 'directory')) - elif not l: - print('No links entered!', color='WARN') - else: - break - except Exception as e: - print('Exception:', e, color='ERROR') - choices = () - - save_to_file = False - - if args.save_magnets or config.getboolean('Save', 'magnets'): - print('Saving selected magnets...') - save_magnets(choices, mags, config.get('Save', 'directory')) - save_to_file = True - - if args.save_torrents or config.getboolean('Save', 'torrents'): - print('Saving selected torrents...') - save_torrents(choices, mags, config.get('Save', 'directory')) - save_to_file = True - - if save_to_file: - return - - for choice in choices: - url = mags[int(choice)][0] - - if args.transmission or config.getboolean('Misc', 'transmission'): - subprocess.call(transmission_command + ['-l', '--add', url], shell=False) - subprocess.call(transmission_command + ['-l']) - - elif args.command or config.get('Misc', 'openCommand'): - command = config.get('Misc', 'openCommand') - if args.command: - command = args.command - subprocess.call(parse_cmd(command, url), shell=False) - - else: - webbrowser.open(url) - - -if __name__ == '__main__': - main() From aea37d38375ef3e24a9377595bd533539ace3d07 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sun, 30 Aug 2015 01:30:38 +0000 Subject: [PATCH 04/13] Add setup.py --- setup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 556dbde..6a3137c 100644 --- a/setup.py +++ b/setup.py @@ -7,11 +7,16 @@ setup(name='pirate-get', author='vikstrous', author_email='', license='GPL', - packages=['pirate-get'], + packages=['pirate'], entry_points={ - 'console_scripts': ['pirate-get = src.main:main'] + 'console_scripts': ['pirate-get = pirate.pirate:main'] }, - keywords=['server'], + install_requires=['colorama'], + keywords=['torrent', 'magnet', 'download', 'tpb', 'client'], classifiers=[ + 'Topic :: Utilities' + 'Topic :: Terminals' + 'Topic :: System :: Networking' + 'Programming Language :: Python :: 3 :: Only' 'License :: OSI Approved :: GNU General Public License (GPL)', ]) \ No newline at end of file From ee8abc5d372e8e3de76b3f92c38132e5b6a26170 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sun, 30 Aug 2015 02:00:06 +0000 Subject: [PATCH 05/13] Fix missing imports --- pirate/pirate.py | 18 +++++++++--------- pirate/print.py | 9 +++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pirate/pirate.py b/pirate/pirate.py index e3ae9dc..15ad1e5 100755 --- a/pirate/pirate.py +++ b/pirate/pirate.py @@ -152,7 +152,7 @@ def main(): if args.list_sorts: cur_color = 'zebra_0' - for key, value in sorted(sorts.items()): + for key, value in sorted(pirate.data.sorts.items()): cur_color = 'zebra_0' if cur_color == 'zebra_1' else 'zebra_1' print(str(value), '\t', key, sep='', color=cur_color) return @@ -257,11 +257,11 @@ def main(): elif code == 'p': pirate.print.search_results(mags, sizes, uploaded) elif code == 'm': - save_magnets(choices, mags, - config.get('Save', 'directory')) + pirate.torrent.save_magnets(choices, mags, config.get( + 'Save', 'directory')) elif code == 't': - save_torrents(choices, mags, - config.get('Save', 'directory')) + pirate.torrent.save_torrents(choices, mags, config.get( + 'Save', 'directory')) elif not l: print('No links entered!', color='WARN') else: @@ -274,14 +274,14 @@ def main(): if args.save_magnets or config.getboolean('Save', 'magnets'): print('Saving selected magnets...') - pirate.torrent.save_magnets(choices, mags, - config.get('Save', 'directory')) + pirate.torrent.save_magnets(choices, mags, config.get( + 'Save', 'directory')) save_to_file = True if args.save_torrents or config.getboolean('Save', 'torrents'): print('Saving selected torrents...') - pirate.torrent.save_torrents(choices, mags, - config.get('Save', 'directory')) + pirate.torrent.save_torrents(choices, mags, config.get( + 'Save', 'directory')) save_to_file = True if save_to_file: diff --git a/pirate/print.py b/pirate/print.py index 7e0f612..f3e2e83 100644 --- a/pirate/print.py +++ b/pirate/print.py @@ -80,9 +80,9 @@ def descriptions(chosen_links, mags, site, identifiers): for link in chosen_links: link = int(link) path = '/torrent/%s/' % identifiers[link] - req = request.Request(site + path, headers=default_headers) + req = request.Request(site + path, headers=pirate.data.default_headers) req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) + f = request.urlopen(req, timeout=pirate.data.default_timeout) if f.info().get('Content-Encoding') == 'gzip': f = gzip.GzipFile(fileobj=BytesIO(f.read())) @@ -105,9 +105,10 @@ def file_lists(chosen_links, mags, site, identifiers): for link in chosen_links: path = '/ajax_details_filelist.php' query = '?id=' + identifiers[int(link)] - req = request.Request(site + path + query, headers=default_headers) + req = request.Request(site + path + query, + headers=pirate.data.default_headers) req.add_header('Accept-encoding', 'gzip') - f = request.urlopen(req, timeout=default_timeout) + f = request.urlopen(req, timeout=pirate.data.default_timeout) if f.info().get('Content-Encoding') == 'gzip': f = gzip.GzipFile(fileobj=BytesIO(f.read())) From 9e227aecfa5b99c064d6895a5d78be1280196812 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sun, 30 Aug 2015 02:00:42 +0000 Subject: [PATCH 06/13] Cosmesis --- pirate/data.py | 2 +- pirate/local.py | 4 ++-- pirate/pirate.py | 28 ++++++++++++++-------------- pirate/print.py | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pirate/data.py b/pirate/data.py index 1076a64..27ca62d 100644 --- a/pirate/data.py +++ b/pirate/data.py @@ -5,4 +5,4 @@ sorts = json.load(open('data/sorts.json')) default_headers = {'User-Agent': 'pirate get'} default_timeout = 10 -colored_output = True \ No newline at end of file +colored_output = True diff --git a/pirate/local.py b/pirate/local.py index 2c2c3f2..2d62382 100644 --- a/pirate/local.py +++ b/pirate/local.py @@ -1,7 +1,7 @@ import urllib.parse as parse import html.parser as parser -# create a subclass and override the handler methods + class BayParser(parser.HTMLParser): title = '' q = '' @@ -38,4 +38,4 @@ def search(db, terms): xml = open(db).readlines() parser = BayParser(' '.join(terms)) parser.feed(''.join(xml)) - return parser.results \ No newline at end of file + return parser.results diff --git a/pirate/pirate.py b/pirate/pirate.py index 15ad1e5..26d4211 100755 --- a/pirate/pirate.py +++ b/pirate/pirate.py @@ -3,7 +3,7 @@ import os import argparse import subprocess import configparser -import socket +import socket import urllib.request as request import urllib.error import webbrowser @@ -30,7 +30,7 @@ def load_config(): config.set('LocalDB', 'enabled', 'false') config.set('LocalDB', 'path', expanduser('~/downloads/pirate-get/db')) - config.add_section('Misc') + config.add_section('Misc') config.set('Misc', 'openCommand', '') config.set('Misc', 'transmission', 'false') config.set('Misc', 'colors', 'true') @@ -58,8 +58,8 @@ def parse_cmd(cmd, url): ret = [i[0].strip().replace('%s', url) for i in ret] ret_no_quotes = [] for item in ret: - if ((item[0] == "'" and item[-1] == "'") or ( - item[0] == '"' and item[-1] == '"')): + if ((item[0] == "'" and item[-1] == "'") or + (item[0] == '"' and item[-1] == '"')): ret_no_quotes.append(item[1:-1]) else: ret_no_quotes.append(item) @@ -70,7 +70,7 @@ def main(): config = load_config() parser = argparse.ArgumentParser( - description='finds and downloads torrents from the Pirate Bay') + description='finds and downloads torrents from the Pirate Bay') parser.add_argument('-b', dest='browse', action='store_true', help='display in Browse mode') @@ -82,7 +82,7 @@ def main(): help='specify a sort option', default='SeedersDsc') parser.add_argument('-R', dest='recent', action='store_true', help='torrents uploaded in the last 48hours.' - '*ignored in searches*') + '*ignored in searches*') parser.add_argument('-l', dest='list_categories', action='store_true', help='list categories') @@ -124,8 +124,7 @@ def main(): args = parser.parse_args() if (config.getboolean('Misc', 'colors') and not args.color - or not config.getboolean('Misc', 'colors')): - #global colored_output + or not config.getboolean('Misc', 'colors')): pirate.data.colored_output = False if args.save_directory: @@ -182,17 +181,18 @@ def main(): for mirror in mirrors: try: print('Trying', mirror, end='... ') - mags, sizes, uploaded, ids = pirate.torrent.remote(args, mirror) + mags, sizes, uploaded, ids = pirate.torrent.remote(args, + mirror) except (urllib.error.URLError, socket.timeout, - IOError, ValueError): + IOError, ValueError): print('Failed', color='WARN') else: site = mirror print('Ok', color='alt') break else: - print('No available mirrors :(', color='WARN') - return + print('No available mirrors :(', color='WARN') + return if not mags: print('No results') @@ -212,8 +212,8 @@ def main(): print("\nSelect links (Type 'h' for more options" ", 'q' to quit)", end='\b', color='alt') try: - l=input(': ') - except KeyboardInterrupt : + l = input(': ') + except KeyboardInterrupt: print('\nCancelled.') return diff --git a/pirate/print.py b/pirate/print.py index f3e2e83..c971e34 100644 --- a/pirate/print.py +++ b/pirate/print.py @@ -58,7 +58,7 @@ def search_results(mags, sizes, uploaded, local): else: no_seeders, no_leechers = map(int, magnet[1:]) size, unit = (float(sizes[m][0]), - sizes[m][1]) if sizes else (0, '???') + sizes[m][1]) if sizes else (0, '???') date = uploaded[m] # compute the S/L ratio (Higher is better) @@ -68,7 +68,7 @@ def search_results(mags, sizes, uploaded, local): ratio = float('inf') line = ('{:4} {:5} {:5} {:5.1f} {:5.1f}' - ' {:3} {:<11} {:{length}}') + ' {:3} {:<11} {:{length}}') content = [m, no_seeders, no_leechers, ratio, size, unit, date, torrent_name[:columns - 52]] @@ -124,4 +124,4 @@ def file_lists(chosen_links, mags, site, identifiers): for f in files: print('{0[0]:>11} {0[1]}'.format(f), color=cur_color) - cur_color = 'zebra_0' if (cur_color == 'zebra_1') else 'zebra_1' \ No newline at end of file + cur_color = 'zebra_0' if (cur_color == 'zebra_1') else 'zebra_1' From 398614704d40a83c01f668d3fdbc503d7fcb167d Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 21:53:50 -0700 Subject: [PATCH 07/13] new tpd tld, blacklist thebay.tv --- pirate/pirate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pirate/pirate.py b/pirate/pirate.py index 26d4211..77413f8 100755 --- a/pirate/pirate.py +++ b/pirate/pirate.py @@ -165,7 +165,7 @@ def main(): sizes, uploaded = [], [] else: - mags, mirrors = [], {'https://thepiratebay.se'} + mags, mirrors = [], {'https://thepiratebay.mn'} try: req = request.Request('https://proxybay.co/list.txt', headers=pirate.data.default_headers) @@ -177,6 +177,8 @@ def main(): raise IOError('The proxy bay responded with an error.') mirrors = mirrors.union([i.decode('utf-8').strip() for i in f.readlines()][3:]) + # This mirror messes up links in the search results page - you need to load a second page to get the magnet link + mirrors.discard('https://thebay.tv') for mirror in mirrors: try: From eb4358831d3ff128b9ac17127a34d45712469edb Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 21:58:01 -0700 Subject: [PATCH 08/13] add email, fix setup.py --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 6a3137c..d8f3c48 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup(name='pirate-get', description='A command line interface for The Pirate Bay', url='https://github.com/vikstrous/pirate-get', author='vikstrous', - author_email='', + author_email='me@viktorstanchev.com', license='GPL', packages=['pirate'], entry_points={ @@ -14,9 +14,9 @@ setup(name='pirate-get', install_requires=['colorama'], keywords=['torrent', 'magnet', 'download', 'tpb', 'client'], classifiers=[ - 'Topic :: Utilities' - 'Topic :: Terminals' - 'Topic :: System :: Networking' - 'Programming Language :: Python :: 3 :: Only' + 'Topic :: Utilities', + 'Topic :: Terminals', + 'Topic :: System :: Networking', + 'Programming Language :: Python :: 3 :: Only', 'License :: OSI Approved :: GNU General Public License (GPL)', - ]) \ No newline at end of file + ]) From ecbd264d02733d5f545959e4530595267f504649 Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 22:20:44 -0700 Subject: [PATCH 09/13] copy 3ce8e304c4ecd5b66531a3d5cafa425802b10311 --- pirate/pirate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pirate/pirate.py b/pirate/pirate.py index 77413f8..eb341f3 100755 --- a/pirate/pirate.py +++ b/pirate/pirate.py @@ -293,8 +293,7 @@ def main(): url = mags[int(choice)][0] if args.transmission or config.getboolean('Misc', 'transmission'): - subprocess.call(transmission_command + ['-l', '--add', url]) - subprocess.call(transmission_command + ['-l']) + subprocess.call(transmission_command + ['--add', url]) elif args.command or config.get('Misc', 'openCommand'): command = config.get('Misc', 'openCommand') @@ -305,6 +304,8 @@ def main(): else: webbrowser.open(url) + if args.transmission or config.getboolean('Misc', 'transmission'): + subprocess.call(transmission_command + ['-l']) if __name__ == '__main__': main() From f60e0bec3f5b5d967fb80cb4e29d9cc67806be17 Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 22:22:45 -0700 Subject: [PATCH 10/13] copy d341813e62df969980c719133bd826ca8654ae04 --- pirate/pirate.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pirate/pirate.py b/pirate/pirate.py index eb341f3..d5c0505 100755 --- a/pirate/pirate.py +++ b/pirate/pirate.py @@ -235,8 +235,20 @@ def main(): # Turn into list l = re.sub(r'^[hdfp, ]*|[hdfp, ]*$', '', l) l = re.sub('[ ,]+', ',', l) - l = re.sub('[^0-9,]', '', l) - choices = l.split(',') + l = re.sub('[^0-9,-]', '', l) + parsed_input = l.split(',') + + # expand ranges + choices = [] + for elem in parsed_input: # loop will generate a list of lists + left, sep, right = elem.partition('-') + if right: + choices.append(list(range(int(left), int(right) + 1))) + else: + choices.append([int(left)]) + choices = sum(choices, []) # flatten list + choices = [str(elem) for elem in choices] # the current code stores the choices as strings instead of ints. not sure if necessary + # Act on option, if supplied print('') From 660d8017706646d3882020257a53f9bdaa6cb754 Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 22:23:23 -0700 Subject: [PATCH 11/13] bump version to 0.2.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d8f3c48..df01a70 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup setup(name='pirate-get', - version='0.2.4', + version='0.2.5', description='A command line interface for The Pirate Bay', url='https://github.com/vikstrous/pirate-get', author='vikstrous', From 17241297f1119962b14ce841bce1f73765ddc84c Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 23:09:51 -0700 Subject: [PATCH 12/13] add example test --- .gitignore | 8 ++- tests/__init__.py | 0 tests/rich.xml | 168 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_local.py | 15 ++++ 4 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 tests/__init__.py create mode 100644 tests/rich.xml create mode 100755 tests/test_local.py diff --git a/.gitignore b/.gitignore index f0b338d..26f2d51 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,10 @@ coverage.xml docs/_build/ # PyBuilder -target/ \ No newline at end of file +target/ + +# vim +*.swp + +# setup.py +publish/* diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/rich.xml b/tests/rich.xml new file mode 100644 index 0000000..5eaa64a --- /dev/null +++ b/tests/rich.xml @@ -0,0 +1,168 @@ + +3211594 +High.Chaparall.S02E02.PDTV.XViD.SWEDiSH-HuBBaTiX +b03c8641415d3a0fc7077f5bf567634442989a74 +375299009 +1 +0 +00 +2004-03-25 23:08:00 +Andra avsnittet på säsong två av High Chaparall. + +2004-04-05 18:56kan nån seeda första avsnittet +2004-05-03 19:18Ja snälla ta och seeda saknar 0,9%. +2004-05-25 18:32Snälla, kan någon seeda?<br /> +1 % kvar :S +2004-05-26 11:28asså har legat på 99% nu i 2 veckor, dryyyyyyyygt!! +2004-05-27 21:14Legat på 99 % ett bra tag nu jag också, vore tacksam om någon kunde seeda +2006-06-22 23:03Er det helt nye afsnit? :D +2006-06-25 17:03Wow, the piratebay has really gone to shit, What happened? +2006-06-26 07:22wth is going on +2006-06-30 07:44INFO av avsnit ????? +2006-06-30 07:44avsnitt +2006-08-22 02:10Uri Geller?<br /> +2008-10-04 18:14Form, I was going to say that :( +2009-01-28 20:49lol first torrent on TPB EVER.<br /> +<br /> +<br /> +wow.<br /> +<br /> +<br /> +u think he could have picked a better name than kbdcb lolololol +2009-03-03 22:59It's the oldest TV Show torrent that hasn't been deleted,<br /> +<br /> +yet +2010-04-11 08:26wow i found that this is the first torrent ever to be uploaded to tpb +2010-07-13 13:59oldest torrent evur +2010-08-04 16:54precisely my thought... :) +2011-01-02 18:57Well I have a torrent that uploaded 09-02 2004.<br /> +And it´s still active :D<br /> +7 Years ftw guys!! +2011-11-26 02:22"...Querido diario íntimo: mi corazón estalló de emoción al descubrir el torrent más viejo en 'The Pirate Bay'. Estoy feliz por comentar esta publicación y ser parte de la historia de TPB. Es como escalar el Everest. Sinceramente, gracias...".<br /> +<br /> +"... Dear diary, my heart burst of excitement to discover the oldest torrent in The Pirate Bay '. I am happy to comment on this book and be part of the history of TPB. It's like climbing Everest. Sincerely, thanks ...".<br /> +<br /> +"... Kära dagbok, mitt hjärta brast av spänning för att upptäcka den äldsta torrent i The Pirate Bay". Jag är glad att kommentera denna bok och vara en del av historien om TPB. Det är som att klättra Everest. Vänliga hälsningar, tack ...".<br /> +<br /> +"... Liebes Tagebuch, mein Herz brach der Aufregung um die älteste torrent in The Pirate Bay" zu entdecken. Ich freue mich auf dieses Buch kommentieren und werden Sie Teil der Geschichte der TPB. Es ist wie Bergsteigen Everest. Mit freundlichen Grüßen, dank ...". +2012-01-09 11:04just find this first torrent on TPB, from the help of manOtor m8, thanks 4 ur work kbdcb :) + + + + +3211609 +School.Of.Rock.PROPER.DVDRip.XviD-DMT +a896f7155237fb27e2eaa06033b5796d7ae84a1d +739308799 +0 +2 +00 +2004-03-26 09:35:17 +OrginalRelease + +2004-04-06 06:24hur fan öppnar jag filmen +2004-04-17 19:42med winrar<br /> +2004-04-18 21:19lite seg i början... men riktigt bra efter ett tag +2004-04-28 21:03Seeda... +2004-05-05 12:30Kan ingen seeda... Ligger på 97%... Skojj... +2004-05-08 09:57hur fixar man filmen när den bara"darrar" bilden likson skakar hela tiden. Någon som vet?? +2004-05-09 22:53"hur fixar man filmen när den bara"darrar" bilden likson skakar hela tiden. Någon som vet?"<br /> +<br /> +precis samma sak för mig, försökt med alla codecs mm. fatar 0:an! någon som vet? +2004-05-10 11:11Ni som lyckats med denna film kan väl höra av er och berätta hur ni gjort! +2004-06-07 00:51SWESUB: <a href="http://www.undertexter.se/index.php?p=subark&id=1148" rel="nofollow" target="_new">http://www.undertexter.se/index.php?p=subark&id=1148</a> +2004-06-22 23:44snälla seeda. +2004-08-04 08:15Nån måste läea mig hur man lägger in subs!!<br /> +Och mitt nero (nyaste) bränner INTE avi filer.... :S<br /> +<br /> +:axe: +2004-09-02 22:04mrmaniac å erikapa Haft samma problem...lösningen heter vlc media player!! spelar upp allt perfect...till och med filer som windows ej kan identifiera=) har tyvärr ej URL...men kolla google... lycka till! +2004-09-05 07:53Om ni ska ha vlc, kika in på <a href="http://www.videolan.org./vlc/" rel="nofollow" target="_new">http://www.videolan.org./vlc/</a> +2004-09-06 21:37Nero som inte bränner avi? Det låter ju asdumt. Hur kommer man på något sånt? Säkert någon jäkla anti-piratgrej. *grr* +2004-09-11 19:56Fyfan så bra film, synd att man inte hade en gitarr att rocka med =(. SEVÄRD! +2004-09-29 05:59Fin kvalite och bra ljud. Hoppas bara att den funkar på DVD-spelarn nu :) +2004-10-09 01:09hmm.. kan inte alla dela med sig mera när dom seedar.. snålt :/ +2004-12-14 16:33kan inte alla seeda när dom laddar ner:/ eller??? +2004-12-14 16:35SEEDAAA DE GÅR TRÖÖÖÖGT LIGGER PÅ 20 kb/s!!!:@ +2004-12-20 20:41Kan inte någon seeda lite mer än 20 kbs +2004-12-21 14:27vlc- player spiller bare filer / filmer i et par sekunder så stopper den . åssen fixer jg dette +2004-12-21 16:44Kan inte någon Seeda alla ligger på 81.8% Finns det inte nåon som vill försöka hålla igång det här så det funkar bra. +2005-01-04 13:10Seeda plz +2005-01-04 13:11jag kan seeda sen men ja vill gärna ha filmen först<br /> +2006-06-22 17:23Seeda! Fast på 99.6% Jag ska hjälpa till om jag får ner filmen! +2006-06-22 23:28seeda förfan sitter på 99.0%<br /> +<br /> +ooooooooooooooorka +2006-06-23 09:19Va faen, jag fattar inget. Allt på hela TPB står med bara 1 seed, inkl. YOP 100. Igår var hela TOP 100 annorlunda med jävla "irish drinking songs" å "simpsons" på första plats. Inget verkar va sig likt eftr. tillslaget. Vad har hänt? +2006-06-24 13:11Please seed. I'm at 93.3%, and have been for three days now.<br /> +And also... piet00piet, please stop spamming your comments. It's annoying. +2006-06-27 20:07why cant i download the movie?<br /> +<br /> +can i have a step by step on how to do it? +2006-06-29 16:37Fan Vad LOL Filmen Var Upp o ner när man spela upp den +2006-07-03 13:25This movie roxorz +2006-07-03 14:32PirateBay used to be good, but something has chnaged, now it sucks! You can not tell how many Seeders there are or no one seems to be seeding. +2006-07-07 18:33Tänkte bara påpeka att gula sidorna finns på internet... och att Pirates of the Carribean: Dead Man Chest finns på IsoHunt nu. <br /> +<br /> +Har inte en susning om kvaliteteten håller på att slanga den själv nu. +2006-07-08 17:42this is a funny movie! Both for kids and adults. +2006-07-09 02:52Good stuff!!! +2011-11-01 18:01< a href="<a href="http://www.imdb.com/title/tt0332379/" rel="nofollow" target="_new">http://www.imdb.com/title/tt0332379/</a>">< IMG SRC = "<a href="http://www.imdb.com/title/tt0332379/" rel="nofollow" target="_new">http://www.imdb.com/title/tt0332379/</a>" >< / a > +2011-11-01 18:04The School Of Rock +2011-11-01 18:29<a href="http://www.imdb.com/title/tt0332379/" rel="nofollow" target="_new">http://www.imdb.com/title/tt0332379/</a> +2011-11-01 18:54< IMG SRC = "<a href="http://www.imdb.com/media/rm3808337152/tt0332379" rel="nofollow" target="_new">http://www.imdb.com/media/rm3808337152/tt0332379</a>" >< / a > +2011-11-01 19:05<a href="LINKhttp://www.imdb.com/title/tt0332379/" rel="nofollow" target="_new">LINKhttp://www.imdb.com/title/tt0332379/</a> +2011-11-01 19:24LINK;<a href="http://www.imdb.com/title/tt0332379/" rel="nofollow" target="_new">http://www.imdb.com/title/tt0332379/</a> +2011-11-01 20:01<a href="http://www.imdb.com/media/rm3808337152/tt0332379" rel="nofollow" target="_new">http://www.imdb.com/media/rm3808337152/tt0332379</a> +2011-11-01 20:36 + + + + +3211623 +Gyllene Tider-Samtliga Hits-SE-2004-WLM +3ebb7aa97076cac0ac1b0812f5e16cf46d5daf41 +127185941 +7 +1 +10 +2004-03-29 09:00:10 +Tanka på + +2004-05-16 06:59Seeda plz!<br /> +Solen skiner och det GT är ett måste :P +2004-05-20 15:43seeda lite te..... plezzz måste ha den här samlingen råkade radera den innan +2004-05-31 11:43kan nån jävel seeda?!?!<br /> +2004-05-31 18:11ligger på 87,6% varför seedar ingen?!?!?!?!?!??! +2004-06-02 07:49Jag såg att det behövdes någon som seedar. Håll till godo. +2004-06-02 16:45tack... har väntat flera dar för att få ner den här +2004-06-13 08:07Fan vaa nice! +2004-06-25 18:28Varför kan jag inte koppla upp mig mot ngn peer!? :evil:<br /> +<br /> +Varenda annan jäkla torrent funkar men inte denna :'( +2004-08-12 12:35TackaR! Detta har jag letat efter +2004-08-19 11:30uh, men förfan, reseeda, så håller jag igång den i ett par veckor!<br /> +<br /> +orka ladda om när man bara har 10% kvar... +2005-04-02 05:23seeda tack! +2006-06-23 17:02We need seeders. I'm stuck at 99.7%. Is there anyone who could seed this? +2006-06-24 22:49Sitter med på 99,7%. +2006-06-26 22:32Seeda för helvete! +2006-06-26 22:346 stycken peers - Alla har 99,7% - Hooray! +2006-06-29 22:35när jag trycker: download this torrent, så kommer den upp i typ en halv sekund, sen försvinner den. <br /> +vad gör jag för fel? +2006-06-30 14:25SEEDA....vilken djävla dum kommentar!<br /> +<br /> +Klart att man seedar...iaf på riktiga trackers LOL<br /> +<br /> +2006-07-01 21:33shyst !! jävligt bra ,,, tack m8 för att du ladda upp den xD. +2006-07-05 13:19Bra att ni seedade nu<br /> +<br /> +Tack så hemskt mycket :) +2006-07-18 14:54SNÄLLA SNÄLLA!! Seeda jag har stannat på 99% sen säkert en vecka... +2012-07-24 23:07I sorted all of the music torrents by upload date and this came up as the oldest one. Unsurprising considering this is a Swedish website.<br /> +<br /> +Per Gessle rocks! Whooooo! + + + diff --git a/tests/test_local.py b/tests/test_local.py new file mode 100755 index 0000000..3d6b8d9 --- /dev/null +++ b/tests/test_local.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +import unittest +import pirate.local +import os + +class TestLocal(unittest.TestCase): + + def test_rich_xml(self): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'rich.xml') + expected = [['magnet:?xt=urn:btih:b03c8641415d3a0fc7077f5bf567634442989a74&dn=High.Chaparall.S02E02.PDTV.XViD.SWEDiSH-HuBBaTiX', '?', '?']] + actual = pirate.local.search(path, ('High',)) + self.assertEqual(actual, expected) + +if __name__ == '__main__': + unittest.main() From f04317ca94757f176e7f6ba8f436a19f64c3d251 Mon Sep 17 00:00:00 2001 From: Viktor Stanchev Date: Sat, 29 Aug 2015 23:14:21 -0700 Subject: [PATCH 13/13] add circle.yml --- circle.yml | 10 ++++++++++ requirements-test.txt | 2 ++ requirements.txt | 0 setup.py | 5 +++-- 4 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 circle.yml create mode 100644 requirements-test.txt create mode 100644 requirements.txt mode change 100644 => 100755 setup.py diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..068fc36 --- /dev/null +++ b/circle.yml @@ -0,0 +1,10 @@ +machine: + python: + version: 3.4.2 +test: + override: + - coverage run -m unittest discover + post: + - mkdir -p $CIRCLE_ARTIFACTS/coverage + - cd /home/ubuntu/pirate-get && coverage html --include=`pwd`* --omit="*/tests/*,*migrations*,*__init__*" + - cp -R /home/ubuntu/pirate-get/htmlcov/* $CIRCLE_ARTIFACTS/coverage diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..13e2a80 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +-r requirements.txt +coverage diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index df01a70..24d9dd5 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ -from setuptools import setup +#!/usr/bin/env python3 +from setuptools import setup, find_packages setup(name='pirate-get', version='0.2.5', @@ -7,7 +8,7 @@ setup(name='pirate-get', author='vikstrous', author_email='me@viktorstanchev.com', license='GPL', - packages=['pirate'], + packages=find_packages(), entry_points={ 'console_scripts': ['pirate-get = pirate.pirate:main'] },