commit 257912a12f862d9f4ca3c5970df57f3a62d29c66 Author: Bora M. ALPER Date: Mon Apr 3 00:11:58 2017 +0400 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf426ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,176 @@ + +# Created by https://www.gitignore.io/api/linux,python,pycharm + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +.idea/ + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# End of https://www.gitignore.io/api/linux,python,pycharm diff --git a/COPYING b/COPYING new file mode 100755 index 0000000..dba13ed --- /dev/null +++ b/COPYING @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.rst b/README.rst new file mode 100755 index 0000000..2f6303a --- /dev/null +++ b/README.rst @@ -0,0 +1,83 @@ +========= +magnetico +========= +*Autonomous (self-hosted) BitTorrent DHT search engine suite.* + +magnetico is the first autonomous (self-hosted) BitTorrent DHT search engine suite that is *designed for end-users*. +The suite consists of two packages: + +* **magneticod:** Autonomous BitTorrent DHT crawler and metadata fetcher. +* **magneticow:** Lightweight web interface for magnetico. + +Both programs, combined together, allows anyone with a decent Internet connection to access the vast amount of torrents +waiting to be discovered within the BitTorrent DHT space, *without relying on any central entity*. + +**magnetico** liberates BitTorrent from the yoke of centralised trackers & web-sites and makes it *truly +decentralised*. Finally! + +Features +======== +- Easy installation & minimal requirements: + + - Python 3.5+ and a few Python packages that is available on PyPI. + - Root access is *not* required to install. +- Near-zero configuration: + + - magneticod works out of the box, and magneticow requires minimal configuration to work with the web server you choose. + - Detailed, step-by-step manual to guide you through the installation. +- No reliance on any centralised entity: + + - **magneticod** crawls the BitTorrent DHT by "going" from one node to another, and fetches the metadata using the nodes without using trackers. +- Resilience: + + - Unlike client-server model that web applications use, P2P networks are *chaotic* and **magneticod** is designed to handle all the operational errors accordingly. + +- High performance implementation: + + - **magneticod** utilizes every bit of your bandwidth to discover as many infohashes & metadata as possible. +- Built-in lightweight web interface: + + - **magneticow** features a lightweight web interface to help you access the database without getting on your way. + +Why? +==== +BitTorrent, being a distributed P2P file sharing protocol, has long suffered because of the centralised entities that +people dependent on for searching torrents (websites) and for discovering other peers (trackers). Introduction of DHT +(distributed hash table) eliminated the need for trackers, allowing peers to discover peers through other peers and to +fetch metadata from the leechers & seeders in the network. **magnetico** is the finishing move that allows users to +search for torrents in the network & removes the need for torrent websites. + +Installation Instructions +========================= + **WARNING:** + + **magnetico** is still under active construction, and is considered *pre-alpha* software. Please use **magnetico** + suite with care and follow the installation instructions carefully to install it & secure the installation. Feel + perfectly free to send bug reports, suggestions, or whatever comes to your mind to send to us through GitHub or + personal e-mail. +\ + + **WARNING:** + + **magnetico** currently does NOT have any filtering system NOR it allows individual torrents to be removed from the + database, and BitTorrent DHT network is full of the materials that are considered illegal in many countries + (violence, pornography, copyright infringing content, and even child-pornography). If you are afraid of the legal + consequences, or simply morally against (indirectly) assisting those content to spread around, follow the + **magneticow** installation instructions carefully to password-protect the web-interface from others. + +1. Install **magneticod** first by following its + `installation instruction `_. +2. Install **magneticow** first by following its + `installation instruction `_. + + +License +======= +All the code is licensed under AGPLv3, unless otherwise stated in the source specific source. See ``COPYING`` file for +the full license text. + +---- + +Dedicated to Cemile Binay, in whose hands I thrived. + +Bora M. ALPER \ No newline at end of file diff --git a/magneticod/README.rst b/magneticod/README.rst new file mode 100644 index 0000000..8b051ae --- /dev/null +++ b/magneticod/README.rst @@ -0,0 +1,147 @@ +========== +magneticod +========== +*Autonomous BitTorrent DHT crawler and metadata fetcher.* + +**magneticod** is the daemon that crawls the BitTorrent DHT network in the background to discover info hashes and +fetches metadata from the peers. It uses SQLite 3 that is built-in your Python 3.x distribution to persist data. + +Installation +============ +Requirements +------------ +- Python 3.5 or above. + + **WARNING:** + + Python 3.6.0 and 3.6.1 suffer from a bug (`issue #29714 `_) that causes + magneticod to fail. As it is an interpreter bug that I have no control on, please make sure that you are not using + any of those Python 3 versions to run magneticod. + +- Decent Internet access (IPv4) + + **magneticod** uses UDP protocol to communicate with the nodes in the DHT network, and TCP to communicate with the + peers while fetching metadata. **Please make sure you have a healthy connection;** you can confirm this by checking at + the *connection status indicator* of your BitTorrent client: if it does not indicate any error, **magneticod** should + just work fine. + +Instructions +------------ +1. Download the latest version of **magneticod** from PyPI using pip3: :: + + pip3 install magneticod + +2. Add installation path to the ``$PATH``; append the following line to your ``~/.bashrc`` :: + + export PATH=$PATH:~/.local/bin + +3. Activate the changes to ``$PATH``: :: + + source ~/.bashrc + +4. Confirm that it is running: :: + + magneticod + + Within maximum 5 minutes (and usually under a minute) **magneticod** will discover a few torrents! This, of course, + depends on your bandwidth, and your network configuration (existence of a firewall, misconfigured NAT, etc.). + +5. *(only for systemd users, skip the rest of the steps and proceed to the* `Using`_ *section if you are not a systemd + user or want to use a different solution)* + + Download the magneticod systemd service file (at + `magneticod/systemd/magneticod.service `_) and change the tilde symbol with + the path of your home directory. For example, if my username is ``bora``, this line :: + + ExecStart=~/.local/bin/magneticod + + should become this: :: + + ExecStart=/home/bora/.local/bin/magneticod + + Here, tilde (``~``) is replaced with ``/home/bora``. Run ``echo ~`` to see the path of your own home directory, if + you do not already know. + +6. Copy the magneticod systemd service file to your local systemd configuration directory: :: + + cp magneticod.service ~/.config/systemd/user/ + + You might need to create intermediate directories (``.config``, ``systemd``, and ``user``) if not exists. + +7. Start **magneticod**: :: + + systemctl --user start magneticod + + **magneticod** should now be running under the supervision of systemd and it should also be automatically started + whenever you boot your machine. + + You can check its status and most recent log entries using the following command: :: + + systemctl --user status magneticod + + To stop **magneticod**, issue the following: :: + + systemctl --user stop magneticod +\ + + **Suggestion:** + + Keep **magneticod** running so that when you finish installing **magneticow**, database will be populated and you + can see some results. + +Using +===== +**magneticod** does not require user interference to operate, once it starts running. Hence, there is no "user manual", +although you should beware of these points: + +1. **Network Usage:** + + **magneticod** does *not* have any built-in rate limiter *yet*, and it will literally suck the hell out of your + bandwidth. Unless you are running **magneticod** on a separate machine dedicated for it, you might want to consider + starting it manually only when network load is low (e.g. when you are at work or sleeping at night). + +2. **Pre-Alpha Bugs:** + + **magneticod** is *supposed* to work "just fine", but as being at pre-alpha stage, it's likely that you might find + some bugs. It will be much appreciated if you can report those bugs, so that **magneticod** can be improved. See the + next sub-section for how to mitigate the issue if you are *not* using systemd. + +Automatic Restarting +-------------------- +Due to minor bugs at this stage of its development, **magneticod** should be supervised by another program to be ensured +that it's running, and should be restarted if not. systemd service file supplied by **magneticod** implements that, +although (if you wish) you can also use a much more primitive approach using GNU screen (which comes pre-installed in +many GNU/Linux distributions): + +1. Start screen session named ``magneticod``: :: + + screen -S magneticod + +2. Run **magneticod** forever: :: + + until magneticod; do echo "restarting..."; sleep 5; done; + + This will keep restarting **magneticod** after five seconds in case if it fails. + +3. Detach the session by pressing Ctrl+A and after Ctrl+D. + +4. If you wish to see the logs, or to kill **magneticod**, ``screen -r magneticod`` will attach the original screen + session back. **magneticod** will exit gracefully upon keyboard interrupt (Ctrl+C) [SIGINT]. + +Database +-------- +**magneticod** uses SQLite 3 that is built-in by default in almost all Python distributions. +`appdirs `_ package is used to determine user data directory, which is often +``~/.local/share/magneticod``. **magneticod** uses write-ahead logging for its database, so there might be multiple +files while it is operating, but ``database.sqlite3`` is *the main database where every torrent metadata is stored*. + +License +======= +All the code is licensed under AGPLv3, unless otherwise stated in the source specific source. See ``COPYING`` file +in ``magnetico`` directory for the full license text. + +---- + +Dedicated to Cemile Binay, in whose hands I thrived. + +Bora M. ALPER diff --git a/magneticod/magneticod/__init__.py b/magneticod/magneticod/__init__.py new file mode 100644 index 0000000..0c2c6c3 --- /dev/null +++ b/magneticod/magneticod/__init__.py @@ -0,0 +1,15 @@ +# magneticod - Autonomous BitTorrent DHT crawler and metadata fetcher. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +__version__ = (0, 1, 0) diff --git a/magneticod/magneticod/__main__.py b/magneticod/magneticod/__main__.py new file mode 100644 index 0000000..8344f9e --- /dev/null +++ b/magneticod/magneticod/__main__.py @@ -0,0 +1,159 @@ +# magneticod - Autonomous BitTorrent DHT crawler and metadata fetcher. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +import collections +import functools +import logging +import selectors +import itertools +import os +import sys +import time +import typing + +import appdirs + +from . import __version__ +from . import bittorrent +from . import dht +from . import persistence + + +TICK_INTERVAL = 1 # in seconds (soft constraint) +# maximum (inclusive) number of active (disposable) peers to fetch the metadata per info hash at the same time: +MAX_ACTIVE_PEERS_PER_INFO_HASH = 5 + + +# Global variables are bad bla bla bla, BUT these variables are used so many times that I think it is justified; else +# the signatures of many functions are literally cluttered. +# +# If you are using a global variable, please always indicate that at the VERY BEGINNING of the function instead of right +# before using the variable for the first time. +selector = selectors.DefaultSelector() +database = None # type: persistence.Database +node = None +peers = collections.defaultdict(list) # type: typing.DefaultDict[dht.InfoHash, typing.List[bittorrent.DisposablePeer]] +# info hashes whose metadata is valid & complete (OR complete but deemed to be corrupt) so do NOT download them again: +complete_info_hashes = set() + + +def main(): + global complete_info_hashes, database, node, peers, selector + + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)8s %(message)s") + logging.info("magneticod v%d.%d.%d started", *__version__) + + # noinspection PyBroadException + try: + path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3") + database = persistence.Database(path) + except: + logging.exception("could NOT connect to the database!") + return 1 + + complete_info_hashes = database.get_complete_info_hashes() + + node = dht.SybilNode() + node.when_peer_found = on_peer_found + + selector.register(node, selectors.EVENT_READ) + + try: + loop() + except KeyboardInterrupt: + logging.critical("Keyboard interrupt received! Exiting gracefully...") + pass + finally: + database.close() + selector.close() + node.shutdown() + for peer in itertools.chain.from_iterable(peers.values()): + peer.shutdown() + + return 0 + + +def on_peer_found(info_hash: dht.InfoHash, peer_address) -> None: + global selector, peers, complete_info_hashes + + if len(peers[info_hash]) > MAX_ACTIVE_PEERS_PER_INFO_HASH or info_hash in complete_info_hashes: + return + + try: + peer = bittorrent.DisposablePeer(info_hash, peer_address) + except ConnectionError: + return + + selector.register(peer, selectors.EVENT_READ | selectors.EVENT_WRITE) + peer.when_metadata_found = on_metadata_found + peer.when_error = functools.partial(on_peer_error, peer, info_hash) + peers[info_hash].append(peer) + + +def on_metadata_found(info_hash: dht.InfoHash, metadata: bytes) -> None: + global complete_info_hashes, database, peers, selector + + succeeded = database.add_metadata(info_hash, metadata) + if not succeeded: + logging.info("Corrupt metadata for %s! Ignoring.", info_hash.hex()) + + # When we fetch the metadata of an info hash completely, shut down all other peers who are trying to do the same. + for peer in peers[info_hash]: + selector.unregister(peer) + peer.shutdown() + del peers[info_hash] + + complete_info_hashes.add(info_hash) + + +def on_peer_error(peer: bittorrent.DisposablePeer, info_hash: dht.InfoHash) -> None: + global peers, selector + peer.shutdown() + peers[info_hash].remove(peer) + selector.unregister(peer) + + +def loop() -> None: + global selector, node, peers + + t0 = time.monotonic() + while True: + keys_and_events = selector.select(timeout=TICK_INTERVAL) + + # Check if it is time to tick + delta = time.monotonic() - t0 + if delta >= TICK_INTERVAL: + if not (delta < 2 * TICK_INTERVAL): + logging.warning("Belated TICK! (Δ = %d)", delta) + + node.on_tick() + + t0 = time.monotonic() + + for key, events in keys_and_events: + if events & selectors.EVENT_READ: + key.fileobj.on_receivable() + if events & selectors.EVENT_WRITE: + key.fileobj.on_sendable() + + # Check for entities that would like to write to their socket + keymap = selector.get_map() + for fd in keymap: + fileobj = keymap[fd].fileobj + if fileobj.would_send(): + selector.modify(fileobj, selectors.EVENT_READ | selectors.EVENT_WRITE) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/magneticod/magneticod/bencode.py b/magneticod/magneticod/bencode.py new file mode 100755 index 0000000..959c526 --- /dev/null +++ b/magneticod/magneticod/bencode.py @@ -0,0 +1,165 @@ +# magneticod - Autonomous BitTorrent DHT crawler and metadata fetcher. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . + + +""" +bencode + +Warning: + Encoders do NOT check for circular objects! (and will NEVER check due to speed concerns). + +TODO: + Add support for integers in scientific notation. (?) + Please do re-write this as a shared C module so that we can gain a H U G E speed & performance gain! + + I M P O R T A N T // U R G E N T + Support bytearrays as well! (Currently, only bytes). +""" + + +import typing + + +Types = typing.Union[int, bytes, list, "KRPCDict"] +KRPCDict = typing.Dict[bytes, Types] + + +def dumps(obj) -> bytes: + try: + return __encode[type(obj)](obj) + except: + raise BencodeEncodingError() + + +def loads(bytes_object: bytes) -> Types: + try: + return __decoders[bytes_object[0]](bytes_object, 0)[0] + except Exception as exc: + raise BencodeDecodingError(exc) + + +def loads2(bytes_object: bytes) -> typing.Tuple[Types, int]: + """ + Returns the bencoded object AND the index where the dump of the decoded object ends (exclusive). In less words: + + dump = b"i12eOH YEAH" + object, i = loads2(dump) + print(">>>", dump[i:]) # OUTPUT: >>> b'OH YEAH' + """ + try: + return __decoders[bytes_object[0]](bytes_object, 0) + except Exception as exc: + raise BencodeDecodingError(exc) + + +def __encode_int(i: int) -> bytes: + # False positive... + return b"i%de" % i + + +def __encode_str(s: typing.ByteString) -> bytes: + return b"%d:%s" % (len(s), s) + + +def __encode_list(l: typing.Sequence) -> bytes: + """ REFERENCE IMPLEMENTATION + s = bytearray() + for obj in l: + s += __encode[type(obj)](obj) + return b"l%se" % (s,) + """ + return b"l%se" % b"".join(__encode[type(obj)](obj) for obj in l) + + +def __encode_dict(d: typing.Dict[typing.ByteString, typing.Any]) -> bytes: + s = bytearray() + # Making sure that the keys are in lexicographical order. + # Source: http://stackoverflow.com/a/7375703/4466589 + items = sorted(d.items(), key=lambda k: (k[0].lower(), k[0])) + for key, value in items: + s += __encode_str(key) + s += __encode[type(value)](value) + return b"d%se" % (s, ) + + +__encode = { + int: __encode_int, + bytes: __encode_str, + bytearray: __encode_str, + list: __encode_list, + dict: __encode_dict +} + + +def __decode_int(b: bytes, start_i: int) -> typing.Tuple[int, int]: + end_i = b.find(b"e", start_i) + assert end_i != -1 + return int(b[start_i + 1: end_i]), end_i + 1 + + +def __decode_str(b: bytes, start_i: int) -> typing.Tuple[bytes, int]: + separator_i = b.find(b":", start_i) + assert separator_i != -1 + length = int(b[start_i: separator_i]) + return b[separator_i + 1: separator_i + 1 + length], separator_i + 1 + length + + +def __decode_list(b: bytes, start_i: int) -> typing.Tuple[list, int]: + list_ = [] + i = start_i + 1 + while b[i] != 101: # 101 = ord(b"e") + item, i = __decoders[b[i]](b, i) + list_.append(item) + return list_, i + 1 + + +def __decode_dict(b: bytes, start_i: int) -> typing.Tuple[dict, int]: + dict_ = {} + + i = start_i + 1 + while b[i] != 101: # 101 = ord(b"e") + # Making sure it's between b"0" and b"9" (incl.) + assert 48 <= b[i] <= 57 + key, end_i = __decode_str(b, i) + dict_[key], i = __decoders[b[end_i]](b, end_i) + + return dict_, i + 1 + + +__decoders = { + ord(b"i"): __decode_int, + ord(b"0"): __decode_str, + ord(b"1"): __decode_str, + ord(b"2"): __decode_str, + ord(b"3"): __decode_str, + ord(b"4"): __decode_str, + ord(b"5"): __decode_str, + ord(b"6"): __decode_str, + ord(b"7"): __decode_str, + ord(b"8"): __decode_str, + ord(b"9"): __decode_str, + ord(b"l"): __decode_list, + ord(b"d"): __decode_dict +} + + +class BencodeEncodingError(Exception): + pass + + +class BencodeDecodingError(Exception): + def __init__(self, original_exc): + super().__init__() + self.original_exc = original_exc diff --git a/magneticod/magneticod/bittorrent.py b/magneticod/magneticod/bittorrent.py new file mode 100644 index 0000000..63b8995 --- /dev/null +++ b/magneticod/magneticod/bittorrent.py @@ -0,0 +1,288 @@ +# magneticod - Autonomous BitTorrent DHT crawler and metadata fetcher. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +import errno +import logging +import hashlib +import math +import socket +import random +import typing + +from . import bencode + + +InfoHash = bytes +PeerAddress = typing.Tuple[str, int] + + +class DisposablePeer: + def __init__(self, info_hash: InfoHash, peer_addr: PeerAddress): + self.__socket = socket.socket() + self.__socket.setblocking(False) + # To reduce the latency: + self.__socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True) + self.__socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, True) + res = self.__socket.connect_ex(peer_addr) + if res != errno.EINPROGRESS: + raise ConnectionError() + + self.__info_hash = info_hash + + self.__incoming_buffer = bytearray() + self.__outgoing_buffer = bytearray() + + self.__bt_handshake_complete = False # BitTorrent Handshake + self.__ext_handshake_complete = False # Extension Handshake + self.__ut_metadata = None # Since we don't know ut_metadata code that remote peer uses... + + self.__metadata_size = None + self.__metadata_received = 0 # Amount of metadata bytes received... + self.__metadata = None + + # To prevent double shutdown + self.__shutdown = False + + # Send the BitTorrent handshake message (0x13 = 19 in decimal, the length of the handshake message) + self.__outgoing_buffer += b"\x13BitTorrent protocol%s%s%s" % ( + b"\x00\x00\x00\x00\x00\x10\x00\x01", + self.__info_hash, + self.__random_bytes(20) + ) + + @staticmethod + def when_error() -> None: + raise NotImplementedError() + + @staticmethod + def when_metadata_found(info_hash: InfoHash, metadata: bytes) -> None: + raise NotImplementedError() + + def on_receivable(self) -> None: + while True: + try: + received = self.__socket.recv(8192) + except BlockingIOError: + break + except ConnectionResetError: + self.when_error() + return + except ConnectionRefusedError: + self.when_error() + return + except OSError: # TODO: check for "no route to host 113" error + self.when_error() + return + + if not received: + self.when_error() + return + + self.__incoming_buffer += received + # Honestly speaking, BitTorrent protocol might be one of the most poorly documented and (not the most but) badly + # designed protocols I have ever seen (I am 19 years old so what I could have seen?). + # + # Anyway, all the messages EXCEPT the handshake are length-prefixed by 4 bytes in network order, BUT the + # size of the handshake message is the 1-byte length prefix + 49 bytes, but luckily, there is only one canonical + # way of handshaking in the wild. + if not self.__bt_handshake_complete: + if len(self.__incoming_buffer) < 68: + # We are still receiving the handshake... + return + + if self.__incoming_buffer[1:20] != b"BitTorrent protocol": + # Erroneous handshake, possibly unknown version... + logging.debug("Erroneous BitTorrent handshake! %s", self.__incoming_buffer[:68]) + self.when_error() + return + + self.__on_bt_handshake(self.__incoming_buffer[:68]) + + self.__bt_handshake_complete = True + self.__incoming_buffer = self.__incoming_buffer[68:] + + while len(self.__incoming_buffer) >= 4: + # Beware that while there are still messages in the incoming queue/buffer, one of previous messages might + # have caused an error that necessitates us to quit. + if self.__shutdown: + break + + length = int.from_bytes(self.__incoming_buffer[:4], "big") + if len(self.__incoming_buffer) - 4 < length: + # Message is still incoming... + return + + self.__on_message(self.__incoming_buffer[4:4+length]) + self.__incoming_buffer = self.__incoming_buffer[4+length:] + + def on_sendable(self) -> None: + while self.__outgoing_buffer: + try: + n_sent = self.__socket.send(self.__outgoing_buffer) + assert n_sent + self.__outgoing_buffer = self.__outgoing_buffer[n_sent:] + except BlockingIOError: + break + except OSError: + # In case -while looping- on_sendable is called after socket is closed (mostly because of an error) + return + + def __on_message(self, message: bytes) -> None: + length = len(message) + + if length < 2: + # An extension message has minimum length of 2. + return + + # Every extension message has BitTorrent Message ID = 20 + if message[0] != 20: + # logging.debug("Message is NOT an EXTension message! %s", message[:200]) + return + + # Extension Handshake has the Extension Message ID = 0 + if message[1] == 0: + self.__on_ext_handshake_message(message[2:]) + return + + # ut_metadata extension messages has the Extension Message ID = 1 (as we arbitrarily decided!) + if message[1] != 1: + logging.debug("Message is NOT an ut_metadata message! %s", message[:200]) + return + + # Okay, now we are -almost- sure that this is an extension message, a kind we are most likely interested in... + self.__on_ext_message(message[2:]) + + def __on_bt_handshake(self, message: bytes) -> None: + """ on BitTorrent Handshake... send the extension handshake! """ + if message[25] != 16: + logging.info("Peer does NOT support the extension protocol") + + msg_dict_dump = bencode.dumps({ + b"m": { + b"ut_metadata": 1 + } + }) + # In case you cannot read_file hex: + # 0x14 = 20 (BitTorrent ID indicating that it's an extended message) + # 0x00 = 0 (Extension ID indicating that it's the handshake message) + self.__outgoing_buffer += b"%s\x14\x00%s" % ( + (2 + len(msg_dict_dump)).to_bytes(4, "big"), + msg_dict_dump + ) + + def __on_ext_handshake_message(self, message: bytes) -> None: + if self.__ext_handshake_complete: + return + + try: + msg_dict = bencode.loads(bytes(message)) + except bencode.BencodeDecodingError: + # One might be tempted to close the connection, but why care? Any DisposableNode will be disposed + # automatically anyway (after a certain amount of time if the metadata is still not complete). + logging.debug("Could NOT decode extension handshake message! %s", message[:200]) + return + + try: + # Just to make sure that the remote peer supports ut_metadata extension: + ut_metadata = msg_dict[b"m"][b"ut_metadata"] + metadata_size = msg_dict[b"metadata_size"] + assert metadata_size > 0 + except (AssertionError, KeyError): + self.when_error() + return + + self.__ut_metadata = ut_metadata + try: + self.__metadata = bytearray(metadata_size) + except MemoryError: + logging.exception("Could not allocate %.1f KiB for the metadata!", metadata_size / 1024) + self.when_error() + self.__metadata_size = metadata_size + self.__ext_handshake_complete = True + + # After the handshake is complete, request all the pieces of metadata + n_pieces = math.ceil(self.__metadata_size / (2 ** 14)) + for piece in range(n_pieces): + self.__request_metadata_piece(piece) + + def __on_ext_message(self, message: bytes) -> None: + try: + msg_dict, i = bencode.loads2(bytes(message)) + except bencode.BencodeDecodingError: + # One might be tempted to close the connection, but why care? Any DisposableNode will be disposed + # automatically anyway (after a certain amount of time if the metadata is still not complete). + logging.debug("Could NOT decode extension message! %s", message[:200]) + return + + try: + msg_type = msg_dict[b"msg_type"] + piece = msg_dict[b"piece"] + except KeyError: + logging.debug("Missing EXT keys! %s", msg_dict) + return + + if msg_type == 1: # data + metadata_piece = message[i:] + self.__metadata[piece * 2**14: piece * 2**14 + len(metadata_piece)] = metadata_piece + self.__metadata_received += len(metadata_piece) + + # self.__metadata += metadata_piece + + # logging.debug("PIECE %d RECEIVED %s", piece, metadata_piece[:200]) + + if self.__metadata_received == self.__metadata_size: + if hashlib.sha1(self.__metadata).digest() == self.__info_hash: + self.when_metadata_found(self.__info_hash, bytes(self.__metadata)) + else: + logging.debug("Invalid Metadata! Ignoring.") + + elif msg_type == 2: # reject + logging.info("Peer rejected us.") + self.when_error() + + def __request_metadata_piece(self, piece: int) -> None: + msg_dict_dump = bencode.dumps({ + b"msg_type": 0, + b"piece": piece + }) + # In case you cannot read_file hex: + # 0x14 = 20 (BitTorrent ID indicating that it's an extended message) + # 0x03 = 3 (Extension ID indicating that it's an ut_metadata message) + self.__outgoing_buffer += b"%b\x14%s%s" % ( + (2 + len(msg_dict_dump)).to_bytes(4, "big"), + self.__ut_metadata.to_bytes(1, "big"), + msg_dict_dump + ) + + def shutdown(self) -> None: + if self.__shutdown: + return + try: + self.__socket.shutdown(socket.SHUT_RDWR) + except OSError: + # OSError might be raised in case the connection to the remote peer fails: nevertheless, when_error should + # be called, and the supervisor will try to shutdown the peer, and ta da: OSError! + pass + self.__socket.close() + self.__shutdown = True + + def would_send(self) -> bool: + return bool(len(self.__outgoing_buffer)) + + def fileno(self) -> int: + return self.__socket.fileno() + + @staticmethod + def __random_bytes(n: int) -> bytes: + return random.getrandbits(n * 8).to_bytes(n, "big") diff --git a/magneticod/magneticod/dht.py b/magneticod/magneticod/dht.py new file mode 100644 index 0000000..cc7d508 --- /dev/null +++ b/magneticod/magneticod/dht.py @@ -0,0 +1,264 @@ +# magneticod - Autonomous BitTorrent DHT crawler and metadata fetcher. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +import array +import collections +import zlib +import logging +import random +import socket +import typing + +from . import bencode + +NodeID = bytes +NodeAddress = typing.Tuple[str, int] +PeerAddress = typing.Tuple[str, int] +InfoHash = bytes + + +BOOTSTRAPPING_NODES = [ + ("router.bittorrent.com", 6881), + ("dht.transmissionbt.com", 6881) +] + + +class SybilNode: + # Maximum number of neighbours (this is a THRESHOLD where, once reached, the search for new neighbours will stop; + # but until then, the total number of neighbours might exceed the threshold). + + def __init__(self): + self.__true_id = self.__random_bytes(20) + + self.__socket = socket.socket(type=socket.SOCK_DGRAM) + self.__socket.bind(("0.0.0.0", 0)) + self.__socket.setblocking(False) + + self.__incoming_buffer = array.array("B", (0 for _ in range(65536))) + self.__outgoing_queue = collections.deque() + + self.__routing_table = {} # type: typing.Dict[NodeID, NodeAddress] + + self.__token_secret = self.__random_bytes(4) + + self.__n_max_neighbours = 2000 + + logging.debug("SybilNode %s initialized!", self.__true_id.hex().upper()) + + @staticmethod + def when_peer_found(info_hash: InfoHash, peer_addr: PeerAddress) -> None: + raise NotImplementedError() + + def on_tick(self) -> None: + self.__bootstrap() + self.__make_neighbours() + self.__routing_table.clear() + + def on_receivable(self) -> None: + buffer = self.__incoming_buffer + while True: + try: + _, addr = self.__socket.recvfrom_into(buffer, 65536) + data = buffer.tobytes() + except BlockingIOError: + break + + # Ignore nodes that uses port 0 (assholes). + if addr[1] == 0: + continue + + try: + message = bencode.loads(data) + except bencode.BencodeDecodingError: + continue + + if type(message.get(b"r")) is dict and type(message[b"r"].get(b"nodes")) is bytes: + self.__on_FIND_NODE_response(message) + elif message.get(b"q") == b"get_peers": + self.__on_GET_PEERS_query(message, addr) + elif message.get(b"q") == b"announce_peer": + self.__on_ANNOUNCE_PEER_query(message, addr) + + def on_sendable(self) -> None: + congestion = None + while True: + try: + addr, data = self.__outgoing_queue.pop() + except IndexError: + break + + try: + self.__socket.sendto(data, addr) + except BlockingIOError: + self.__outgoing_queue.appendleft((addr, data)) + break + except PermissionError: + # This exception (EPERM errno: 1) is kernel's way of saying that "you are far too fast, chill". + # It is also likely that we have received a ICMP source quench packet (meaning, that we really need to + # slow down. + # + # Read more here: http://www.archivum.info/comp.protocols.tcp-ip/2009-05/00088/UDP-socket-amp-amp-sendto + # -amp-amp-EPERM.html + congestion = True + break + except OSError: + # Pass in case of trying to send to port 0 (it is much faster to catch exceptions than using an + # if-statement). + pass + + if congestion: + self.__outgoing_queue.clear() + # In case of congestion, decrease the maximum number of nodes to the 90% of the current value. + if self.__n_max_neighbours < 200: + logging.warning("Maximum number of neighbours are now less than 200 due to congestion!") + else: + self.__n_max_neighbours = self.__n_max_neighbours * 9 // 10 + else: + # In case of the lack of congestion, increase the maximum number of nodes by 1%. + self.__n_max_neighbours = self.__n_max_neighbours * 101 // 100 + + def would_send(self) -> bool: + """ Whether node is waiting to write on its socket or not. """ + return bool(self.__outgoing_queue) + + def shutdown(self) -> None: + self.__socket.close() + + def __on_FIND_NODE_response(self, message: bencode.KRPCDict) -> None: + try: + nodes_arg = message[b"r"][b"nodes"] + assert type(nodes_arg) is bytes and len(nodes_arg) % 26 == 0 + except (TypeError, KeyError, AssertionError): + return + + try: + nodes = self.__decode_nodes(nodes_arg) + except AssertionError: + return + + # Add new found nodes to the routing table, assuring that we have no more than n_max_neighbours in total. + if len(self.__routing_table) < self.__n_max_neighbours: + self.__routing_table.update(nodes) + + def __on_GET_PEERS_query(self, message: bencode.KRPCDict, addr: NodeAddress) -> None: + try: + transaction_id = message[b"t"] + assert type(transaction_id) is bytes and transaction_id + info_hash = message[b"a"][b"info_hash"] + assert type(info_hash) is bytes and len(info_hash) == 20 + except (TypeError, KeyError, AssertionError): + return + + # appendleft to prioritise GET_PEERS responses as they are the most fruitful ones! + self.__outgoing_queue.append((addr, bencode.dumps({ + b"y": b"r", + b"t": transaction_id, + b"r": { + b"id": info_hash[:15] + self.__true_id[:5], + b"nodes": b"", + b"token": self.__calculate_token(addr, info_hash) + } + }))) + + def __on_ANNOUNCE_PEER_query(self, message: bencode.KRPCDict, addr: NodeAddress) -> None: + try: + node_id = message[b"a"][b"id"] + assert type(node_id) is bytes and len(node_id) == 20 + transaction_id = message[b"t"] + assert type(transaction_id) is bytes and transaction_id + token = message[b"a"][b"token"] + assert type(token) is bytes + info_hash = message[b"a"][b"info_hash"] + assert type(info_hash) is bytes and len(info_hash) == 20 + if b"implied_port" in message[b"a"]: + implied_port = message[b"a"][b"implied_port"] + assert implied_port in (0, 1) + else: + implied_port = None + port = message[b"a"][b"port"] + + assert type(port) is int and 0 < port < 65536 + except (TypeError, KeyError, AssertionError): + return + + self.__outgoing_queue.append((addr, bencode.dumps({ + b"y": b"r", + b"t": transaction_id, + b"r": { + b"id": node_id[:15] + self.__true_id[:5] + } + }))) + + if implied_port: + peer_addr = (addr[0], addr[1]) + else: + peer_addr = (addr[0], port) + + self.when_peer_found(info_hash, peer_addr) + + def fileno(self) -> int: + return self.__socket.fileno() + + def __bootstrap(self) -> None: + for addr in BOOTSTRAPPING_NODES: + self.__outgoing_queue.append((addr, bencode.dumps({ + b"y": b"q", + b"q": b"find_node", + b"t": self.__random_bytes(2), + b"a": { + b"id": self.__true_id, + b"target": self.__random_bytes(20) + } + }))) + + def __make_neighbours(self) -> None: + for node_id, addr in self.__routing_table.items(): + self.__outgoing_queue.append((addr, bencode.dumps({ + b"y": b"q", + b"q": b"find_node", + b"t": self.__random_bytes(2), + b"a": { + b"id": node_id[:15] + self.__true_id[:5], + b"target": self.__random_bytes(20) + } + }))) + + @staticmethod + def __decode_nodes(infos: bytes) -> typing.List[typing.Tuple[NodeID, NodeAddress]]: + """ REFERENCE IMPLEMENTATION + nodes = [] + for i in range(0, len(infos), 26): + info = infos[i: i + 26] + node_id = info[:20] + node_host = socket.inet_ntoa(info[20:24]) + node_port = int.from_bytes(info[24:], "big") + nodes.append((node_id, (node_host, node_port))) + return nodes + """ + + """ Optimized Version """ + inet_ntoa = socket.inet_ntoa + int_from_bytes = int.from_bytes + return [ + (infos[i:i+20], (inet_ntoa(infos[i+20:i+24]), int_from_bytes(infos[i+24:i+26], "big"))) + for i in range(0, len(infos), 26) + ] + + def __calculate_token(self, addr: NodeAddress, info_hash: InfoHash): + # Believe it or not, faster than using built-in hash (including conversion from int -> bytes of course) + return zlib.adler32(b"%s%s%d%s" % (self.__token_secret, socket.inet_aton(addr[0]), addr[1], info_hash)) + + @staticmethod + def __random_bytes(n: int) -> bytes: + return random.getrandbits(n * 8).to_bytes(n, "big") diff --git a/magneticod/magneticod/persistence.py b/magneticod/magneticod/persistence.py new file mode 100644 index 0000000..168fd43 --- /dev/null +++ b/magneticod/magneticod/persistence.py @@ -0,0 +1,132 @@ +# magneticod - Autonomous BitTorrent DHT crawler and metadata fetcher. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +import logging +import sqlite3 +import time +import typing +import os + +from . import bencode + + +# threshold for pending info hashes before being committed to database: +PENDING_INFO_HASHES = 10 + + +class Database: + def __init__(self, database) -> None: + self.__db_conn = self.__connect(database) + + # We buffer metadata to flush many entries at once, for performance reasons. + # list of tuple (info_hash, name, total_size, discovered_on) + self.__pending_metadata = [] # type: typing.List[typing.Tuple[bytes, str, int, int]] + # list of tuple (info_hash, size, path) + self.__pending_files = [] # type: typing.List[typing.Tuple[bytes, int, bytes]] + + @staticmethod + def __connect(database) -> sqlite3.Connection: + os.makedirs(os.path.split(database)[0], exist_ok=True) + db_conn = sqlite3.connect(database, isolation_level=None) + + db_conn.execute("PRAGMA journal_mode=WAL;") + db_conn.execute("PRAGMA foreign_keys=ON;") + + with db_conn: + db_conn.execute("CREATE TABLE IF NOT EXISTS torrents (" + "id INTEGER PRIMARY KEY," + "info_hash BLOB NOT NULL UNIQUE," + "name TEXT NOT NULL," + "total_size INTEGER NOT NULL CHECK(total_size > 0)," + "discovered_on INTEGER NOT NULL CHECK(discovered_on > 0)" + ");") + db_conn.execute("CREATE TABLE IF NOT EXISTS files (" + "id INTEGER PRIMARY KEY," + "torrent_id INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT," + "size INTEGER NOT NULL," + "path TEXT NOT NULL" + ");") + + return db_conn + + def add_metadata(self, info_hash: bytes, metadata: bytes) -> bool: + files = [] + discovered_on = int(time.time()) + try: + info = bencode.loads(metadata) # type: dict + + assert b"/" not in info[b"name"] + name = info[b"name"].decode("utf-8") + + if b"files" in info: # Multiple File torrent: + for file in info[b"files"]: + assert type(file[b"length"]) is int + # Refuse trailing slash in any of the path items + assert not any(b"/" in item for item in file[b"path"]) + path = "/".join(i.decode("utf-8") for i in file[b"path"]) + files.append((info_hash, file[b"length"], path)) + else: # Single File torrent: + assert type(info[b"length"]) is int + files.append((info_hash, info[b"length"], name)) + # TODO: Make sure this catches ALL, AND ONLY operational errors + except (bencode.BencodeDecodingError, AssertionError, KeyError, AttributeError, UnicodeDecodeError): + return False + + self.__pending_metadata.append((info_hash, name, sum(f[1] for f in files), discovered_on)) + self.__pending_files += files + + logging.info("Added: `%s`", name) + + # Automatically check if the buffer is full, and commit to the SQLite database if so. + if len(self.__pending_metadata) >= PENDING_INFO_HASHES: + self.__commit_metadata() + + return True + + def get_complete_info_hashes(self) -> typing.Set[bytes]: + cur = self.__db_conn.cursor() + try: + cur.execute("SELECT info_hash FROM torrents;") + return set(x[0] for x in cur.fetchall()) + finally: + cur.close() + + def __commit_metadata(self) -> None: + cur = self.__db_conn.cursor() + cur.execute("BEGIN;") + # noinspection PyBroadException + try: + cur.executemany( + "INSERT INTO torrents (info_hash, name, total_size, discovered_on) VALUES (?, ?, ?, ?);", + self.__pending_metadata + ) + cur.executemany( + "INSERT INTO files (torrent_id, size, path) " + "VALUES ((SELECT id FROM torrents WHERE info_hash=?), ?, ?);", + self.__pending_files + ) + cur.execute("COMMIT;") + logging.debug("%d metadata (%d files) are committed to the database.", + len(self.__pending_metadata), len(self.__pending_files)) + self.__pending_metadata.clear() + self.__pending_files.clear() + except: + cur.execute("ROLLBACK;") + logging.exception("Could NOT commit metadata to the database! (%d metadata are pending)", + len(self.__pending_metadata)) + finally: + cur.close() + + def close(self) -> None: + self.__db_conn.close() diff --git a/magneticod/setup.py b/magneticod/setup.py new file mode 100644 index 0000000..29b90d1 --- /dev/null +++ b/magneticod/setup.py @@ -0,0 +1,38 @@ +from setuptools import setup + + +def read_file(path): + with open(path) as file: + return file.read() + + +setup( + name="magneticod", + version="0.1.0", + description="Autonomous BitTorrent DHT crawler and metadata fetcher.", + long_description=read_file("README.rst"), + url="http://magnetico.org", + author="Mert Bora ALPER", + author_email="bora@boramalper.org", + license="GNU Affero General Public License v3 or later (AGPLv3+)", + packages=["magneticod"], + zip_safe=False, + entry_points={ + "console_scripts": ["magneticod=magneticod.__main__:main"] + }, + + install_requires=[ + "appdirs>=1.4.3" + ], + + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Environment :: No Input/Output (Daemon)", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + ] +) diff --git a/magneticod/systemd/magneticod.service b/magneticod/systemd/magneticod.service new file mode 100644 index 0000000..dd1a98d --- /dev/null +++ b/magneticod/systemd/magneticod.service @@ -0,0 +1,10 @@ +[Unit] +Description=magneticod: autonomous BitTorrent DHT crawler and metadata fetcher + +[Service] +ExecStart=~/.local/bin/magneticod +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/magneticow/MANIFEST.in b/magneticow/MANIFEST.in new file mode 100644 index 0000000..61d4d26 --- /dev/null +++ b/magneticow/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include magneticow/static * +recursive-include magneticow/templates * diff --git a/magneticow/README.rst b/magneticow/README.rst new file mode 100644 index 0000000..28594d9 --- /dev/null +++ b/magneticow/README.rst @@ -0,0 +1,153 @@ +========== +magneticow +========== +*Lightweight web interface for magnetico.* + +**magneticow** is a lightweight web interface to search and to browse the torrents that its counterpart (**magneticod**) +discovered. It allows fast full text search of the names of the torrents, by correctly parsing them into their elements. + +Installation +============ +**magneticow** uses `gevent `_ as a "standalone WSGI container" (you can think of it as an +embedded HTTP server), and connects to the same SQLite 3 database that **magneticod** writes. Hence, **root or sudo +access is NOT required at any stage, during or after the installation process.** + +Requirements +------------ +- Python 3.5 or above. + +Instructions +------------ + **WARNING:** + + **magnetico** currently does NOT have any filtering system NOR it allows individual torrents to be removed from the + database, and BitTorrent DHT network is full of the materials that are considered illegal in many countries + (violence, pornography, copyright infringing content, and even child-pornography). If you are afraid of the legal + consequences, or simply morally against (indirectly) assisting those content to spread around, follow the + **magneticow** installation instructions carefully to password-protect the web-interface from others. +\ + + **WARNING:** + + **magneticow** is *NOT* designed to scale, and will fail miserably if you try to use it like a public torrent + website. This is a *deliberate* technical decision, not a bug or something to be fixed; another web interface with + more features to support such use cases and scalability *might* be developed, but **magneticow** will NEVER be the + case. + +1. Download the latest version of **magneticow** from PyPI: :: + + pip3 install magneticow + +2. Add installation path to the ``$PATH``; append the following line to your ``~/.bashrc`` *(you can skip to step 4 if + you installed magneticod first as advised)* :: + + export PATH=$PATH:~/.local/bin + +3. Activate the changes to ``$PATH``: :: + + source ~/.bashrc + +4. Confirm that it is running: :: + + magneticow --port 8080 --user username_1 password_1 --user username_2 password_2 + + Do not forget to actually visit the website, and run a search without any keywords (i.e. simply press the enter + button); this should return a list of most recently discovered torrents. If **magneticod** has not been running long + enough, database might be completely empty and you might see no results (5 minutes should suffice to discover more + than a dozen torrents). + +5. *(only for systemd users, skip the rest of the steps and proceed to the* `Using`_ *section if you are not a systemd + user or want to use a different solution)* + + Download the magneticow systemd service file (at + `magneticow/systemd/magneticow.service `_) and expand the tilde symbol with the path of + your home directory. Also, choose a port (> 1024) for **magneticow** to listen on, and supply username(s) and + password(s). + + For example, if my home directory is ``/home/bora``, and I want to create two users named ``bora`` and ``tolga`` with + passwords ``staatsangehörigkeit`` and ``bürgerschaft``, and then **magneticow** to listen on port 8080, this line :: + + ExecStart=~/.local/bin/magneticow --port PORT --user USERNAME PASSWORD + + should become this: :: + + ExecStart=/home/bora/.local/bin/magneticow --port 8080 --user bora staatsangehörigkeit --user tolga bürgerschaft + + Run ``echo ~`` to see the path of your own home directory, if you do not already know. + + **WARNING:** + + **At least one username and password MUST be supplied.** This is to protect the privacy of your **magneticow** + installation, although **beware that this does NOT encrypt the communication between your browser and the + server!** + +6. Copy the magneticow systemd service file to your local systemd configuration directory: :: + + cp magneticow.service ~/.config/systemd/user/ + +7. Start **magneticow**: :: + + systemctl --user start magneticow + + **magneticow** should now be running under the supervision of systemd and it should also be automatically started + whenever you boot your machine. + + You can check its status and most recent log entries using the following command: :: + + systemctl --user status magneticow + + To stop **magneticow**, issue the following: :: + + systemctl --user stop magneticow + +Using +===== +**magneticow** does not require user interference to operate, once it starts running. Hence, there is no "user manual", +although you should beware of these points: + +1. **Resource Usage:** + + To repeat it for the last time, **magneticow** is a lightweight web interface for magnetico and is not suitable for + handling many users simultaneously. Misusing **magneticow** will likely to lead high processor usage and increased + response times. If that is the case, you might consider lowering the priority of **magneticow** using ``renice`` + command. + +2. **Pre-Alpha Bugs:** + + **magneticow** is *supposed* to work "just fine", but as being at pre-alpha stage, it's likely that you might find + some bugs. It will be much appreciated if you can report those bugs, so that **magneticow** can be improved. See the + next sub-section for how to mitigate the issue if you are *not* using systemd. + +Automatic Restarting +-------------------- +Due to minor bugs at this stage of its development, **magneticow** should be supervised by another program to be ensured +that it's running, and should be restarted if not. systemd service file supplied by **magneticow** implements that, +although (if you wish) you can also use a much more primitive approach using GNU screen (which comes pre-installed in +many GNU/Linux distributions): + +1. Start screen session named ``magneticow``: :: + + screen -S magneticow + +2. Run **magneticow** forever: :: + + until magneticow; do echo "restarting..."; sleep 5; done; + + This will keep restarting **magneticow** after five seconds in case if it fails. + +3. Detach the session by pressing Ctrl+A and after Ctrl+D. + +4. If you wish to see the logs, or to kill **magneticow**, ``screen -r magneticow`` will attach the original screen + session back. **magneticow** will exit gracefully upon keyboard interrupt (Ctrl+C) [SIGINT]. + +License +======= +All the code is licensed under AGPLv3, unless otherwise stated in the source specific source. See ``COPYING`` file +in ``magnetico`` directory for the full license text. + + +---- + +Dedicated to Cemile Binay, in whose hands I thrived. + +Bora M. ALPER \ No newline at end of file diff --git a/magneticow/magneticow/__init__.py b/magneticow/magneticow/__init__.py new file mode 100644 index 0000000..1234a18 --- /dev/null +++ b/magneticow/magneticow/__init__.py @@ -0,0 +1,14 @@ +# magneticow - Lightweight web interface for magnetico. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . diff --git a/magneticow/magneticow/__main__.py b/magneticow/magneticow/__main__.py new file mode 100644 index 0000000..ebac3c4 --- /dev/null +++ b/magneticow/magneticow/__main__.py @@ -0,0 +1,74 @@ +# magneticow - Lightweight web interface for magnetico. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . + +import argparse +import sys +import textwrap + +import gevent.wsgi + +from magneticow import magneticow + + +def main() -> int: + arguments = parse_args() + magneticow.app.arguments = arguments + + http_server = gevent.wsgi.WSGIServer(("", arguments.port), magneticow.app) + + try: + http_server.serve_forever() + except KeyboardInterrupt: + return 0 + + return 1 + + +def parse_args() -> dict: + parser = argparse.ArgumentParser( + description="Lightweight web interface for magnetico.", + epilog=textwrap.dedent("""\ + Copyright (C) 2017 Mert Bora ALPER + Dedicated to Cemile Binay, in whose hands I thrived. + + This program is free software: you can redistribute it and/or modify it under + the terms of the GNU Affero General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) any + later version. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + PARTICULAR PURPOSE. See the GNU Affero General Public License for more + details. + + You should have received a copy of the GNU Affero General Public License along + with this program. If not, see . + """), + allow_abbrev=False, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--port", action="store", type=int, required=True, + help="the port number magneticow web server should listen on" + ) + parser.add_argument( + "--user", action="append", nargs=2, metavar=("USERNAME", "PASSWORD"), type=str, required=True, + help="the pair(s) of username and password for basic HTTP authentication" + ) + + return parser.parse_args(sys.argv[1:]) + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/magneticow/magneticow/magneticow.py b/magneticow/magneticow/magneticow.py new file mode 100644 index 0000000..7cf7c80 --- /dev/null +++ b/magneticow/magneticow/magneticow.py @@ -0,0 +1,234 @@ +# magneticow - Lightweight web interface for magnetico. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +import collections +import functools +from datetime import datetime +import sqlite3 +import os + +import appdirs +import flask + +from magneticow import utils + + +File = collections.namedtuple("file", ["path", "size"]) +Torrent = collections.namedtuple("torrent", ["info_hash", "name", "size", "discovered_on", "files"]) + + +app = flask.Flask(__name__) +app.config.from_object(__name__) + + +# Adapted from: http://flask.pocoo.org/snippets/8/ +# (c) Copyright 2010 - 2017 by Armin Ronacher +# BEGINNING OF THE COPYRIGHTED CONTENT +def check_auth(supplied_username, supplied_password): + """ This function is called to check if a username / password combination is valid. """ + for username, password in app.arguments.user: + if supplied_username == username and supplied_password == password: + return True + return False + + +def authenticate(): + """ Sends a 401 response that enables basic auth. """ + return flask.Response( + "Could not verify your access level for that URL.\n" + "You have to login with proper credentials", + 401, + {"WWW-Authenticate": 'Basic realm="Login Required"'} + ) + + +def requires_auth(f): + @functools.wraps(f) + def decorated(*args, **kwargs): + auth = flask.request.authorization + if not auth or not check_auth(auth.username, auth.password): + return authenticate() + return f(*args, **kwargs) + return decorated +# END OF THE COPYRIGHTED CONTENT + + +@app.route("/") +@requires_auth +def home_page(): + return flask.render_template("homepage.html") + + +@app.route("/torrents/") +@requires_auth +def torrents(): + if flask.request.args: + if flask.request.args["search"] == "": + return newest_torrents() + return search_torrents() + else: + return newest_torrents() + + +def search_torrents(): + magneticod_db = get_magneticod_db() + + search = flask.request.args["search"] + page = int(flask.request.args.get("page", 0)) + + context = { + "search": search, + "page": page + } + + with magneticod_db: + cur = magneticod_db.execute( + "SELECT" + " info_hash, " + " name, " + " total_size, " + " discovered_on " + "FROM torrents " + "INNER JOIN (" + " SELECT torrent_id, rank(matchinfo(fts_torrents, 'pcnxal')) AS rank " + " FROM fts_torrents " + " WHERE name MATCH ? " + " ORDER BY rank ASC" + " LIMIT 20 OFFSET ?" + ") AS ranktable ON torrents.id=ranktable.torrent_id;", + (search, 20 * page) + ) + context["torrents"] = [Torrent(t[0].hex(), t[1], utils.to_human_size(t[2]), + datetime.fromtimestamp(t[3]).strftime("%d/%m/%Y"), []) + for t in cur.fetchall()] + + if len(context["torrents"]) < 20: + context["next_page_exists"] = False + else: + context["next_page_exists"] = True + + return flask.render_template("torrents.html", **context) + + +def newest_torrents(): + magneticod_db = get_magneticod_db() + + page = int(flask.request.args.get("page", 0)) + + context = { + "page": page + } + + with magneticod_db: + cur = magneticod_db.execute( + "SELECT " + " info_hash, " + " name, " + " total_size, " + " discovered_on " + "FROM torrents " + "ORDER BY discovered_on DESC LIMIT 20 OFFSET ?", + (20 * page,) + ) + context["torrents"] = [Torrent(t[0].hex(), t[1], utils.to_human_size(t[2]), datetime.fromtimestamp(t[3]).strftime("%d/%m/%Y"), []) + for t in cur.fetchall()] + + # noinspection PyTypeChecker + if len(context["torrents"]) < 20: + context["next_page_exists"] = False + else: + context["next_page_exists"] = True + + return flask.render_template("torrents.html", **context) + + +@app.route("/torrents//", defaults={"name": None}) +@requires_auth +def torrent_redirect(**kwargs): + magnetico_db = get_magneticod_db() + + try: + info_hash = bytes.fromhex(kwargs["info_hash"]) + assert len(info_hash) == 20 + except (AssertionError, ValueError): # In case info_hash variable is not a proper hex-encoded bytes + return flask.abort(400) + + with magnetico_db: + cur = magnetico_db.execute("SELECT name FROM torrents WHERE info_hash=? LIMIT 1;", (info_hash,)) + try: + name = cur.fetchone()[0] + except TypeError: # In case no results returned, TypeError will be raised when we try to subscript None object + return flask.abort(404) + + return flask.redirect("/torrents/%s/%s" % (kwargs["info_hash"], name), code=301) + + +@app.route("/torrents//") +@requires_auth +def torrent(**kwargs): + magneticod_db = get_magneticod_db() + context = {} + + try: + info_hash = bytes.fromhex(kwargs["info_hash"]) + assert len(info_hash) == 20 + except (AssertionError, ValueError): # In case info_hash variable is not a proper hex-encoded bytes + return flask.abort(400) + + with magneticod_db: + cur = magneticod_db.execute("SELECT name, discovered_on FROM torrents WHERE info_hash=? LIMIT 1;", (info_hash,)) + try: + name, discovered_on = cur.fetchone() + except TypeError: # In case no results returned, TypeError will be raised when we try to subscript None object + return flask.abort(404) + + cur = magneticod_db.execute("SELECT path, size FROM files " + "WHERE torrent_id=(SELECT id FROM torrents WHERE info_hash=? LIMIT 1);", + (info_hash,)) + raw_files = cur.fetchall() + size = sum(f[1] for f in raw_files) + files = [File(f[0], utils.to_human_size(f[1])) for f in raw_files] + + context["torrent"] = Torrent(info_hash.hex(), name, utils.to_human_size(size), datetime.fromtimestamp(discovered_on).strftime("%d/%m/%Y"), files) + + return flask.render_template("torrent.html", **context) + + +def get_magneticod_db(): + """ Opens a new database connection if there is none yet for the current application context. """ + if hasattr(flask.g, "magneticod_db"): + return flask.g.magneticod_db + + magneticod_db_path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3") + magneticod_db = flask.g.magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None) + + with magneticod_db: + magneticod_db.execute("CREATE VIRTUAL TABLE temp.fts_torrents USING fts4(torrent_id INTEGER, name TEXT NOT NULL);") + magneticod_db.execute("INSERT INTO fts_torrents (torrent_id, name) SELECT id, name FROM torrents;") + magneticod_db.execute("INSERT INTO fts_torrents (fts_torrents) VALUES ('optimize');") + + magneticod_db.execute("CREATE TEMPORARY TRIGGER on_torrents_insert AFTER INSERT ON torrents FOR EACH ROW BEGIN" + " INSERT INTO fts_torrents (torrent_id, name) VALUES (NEW.id, NEW.name);" + "END;") + + magneticod_db.create_function("rank", 1, utils.rank) + + return magneticod_db + + +@app.teardown_appcontext +def close_magneticod_db(error): + """ Closes the database again at the end of the request. """ + if hasattr(flask.g, "magneticod_db"): + flask.g.magneticod_db.close() diff --git a/magneticow/magneticow/static/assets/magnet.gif b/magneticow/magneticow/static/assets/magnet.gif new file mode 100644 index 0000000..346763b Binary files /dev/null and b/magneticow/magneticow/static/assets/magnet.gif differ diff --git a/magneticow/magneticow/static/fonts/NotoMono/LICENSE_OFL.txt b/magneticow/magneticow/static/fonts/NotoMono/LICENSE_OFL.txt new file mode 100644 index 0000000..d952d62 --- /dev/null +++ b/magneticow/magneticow/static/fonts/NotoMono/LICENSE_OFL.txt @@ -0,0 +1,92 @@ +This Font Software is licensed under the SIL Open Font License, +Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/magneticow/magneticow/static/fonts/NotoMono/Regular.ttf b/magneticow/magneticow/static/fonts/NotoMono/Regular.ttf new file mode 100644 index 0000000..3560a3a Binary files /dev/null and b/magneticow/magneticow/static/fonts/NotoMono/Regular.ttf differ diff --git a/magneticow/magneticow/static/fonts/NotoSansUI/Bold.ttf b/magneticow/magneticow/static/fonts/NotoSansUI/Bold.ttf new file mode 100644 index 0000000..6dee5c2 Binary files /dev/null and b/magneticow/magneticow/static/fonts/NotoSansUI/Bold.ttf differ diff --git a/magneticow/magneticow/static/fonts/NotoSansUI/BoldItalic.ttf b/magneticow/magneticow/static/fonts/NotoSansUI/BoldItalic.ttf new file mode 100644 index 0000000..4bd6f9d Binary files /dev/null and b/magneticow/magneticow/static/fonts/NotoSansUI/BoldItalic.ttf differ diff --git a/magneticow/magneticow/static/fonts/NotoSansUI/Italic.ttf b/magneticow/magneticow/static/fonts/NotoSansUI/Italic.ttf new file mode 100644 index 0000000..54cae82 Binary files /dev/null and b/magneticow/magneticow/static/fonts/NotoSansUI/Italic.ttf differ diff --git a/magneticow/magneticow/static/fonts/NotoSansUI/LICENSE_OFL.txt b/magneticow/magneticow/static/fonts/NotoSansUI/LICENSE_OFL.txt new file mode 100644 index 0000000..d952d62 --- /dev/null +++ b/magneticow/magneticow/static/fonts/NotoSansUI/LICENSE_OFL.txt @@ -0,0 +1,92 @@ +This Font Software is licensed under the SIL Open Font License, +Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/magneticow/magneticow/static/fonts/NotoSansUI/Regular.ttf b/magneticow/magneticow/static/fonts/NotoSansUI/Regular.ttf new file mode 100644 index 0000000..65b29fc Binary files /dev/null and b/magneticow/magneticow/static/fonts/NotoSansUI/Regular.ttf differ diff --git a/magneticow/magneticow/static/scripts/torrent.js b/magneticow/magneticow/static/scripts/torrent.js new file mode 100644 index 0000000..43578c3 --- /dev/null +++ b/magneticow/magneticow/static/scripts/torrent.js @@ -0,0 +1,59 @@ +// Derived from the Source: http://stackoverflow.com/a/17141374 +// Copyright (c) 2013 'icktoofay' on stackoverflow.com +// Licensed under Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0) +// See https://creativecommons.org/licenses/by-sa/3.0/ for details of the license. +// See https://meta.stackexchange.com/questions/271080/the-mit-license-clarity-on-using-code-on-stack-overflow-and-stack-exchange +// for legal concerns "on using code on Stack Overflow and Stack Exchange". + +"use strict"; + + +window.onload = function() { + var pre_element = document.getElementsByTagName("pre")[0]; + var paths = pre_element.textContent.replace(/\s+$/, "").split("\n"); + paths = paths.map(function(path) { return path.split('/'); }); + pre_element.textContent = stringify(structurise(paths)).join("\n"); +}; + + +function structurise(paths) { + var items = []; + for(var i = 0, l = paths.length; i < l; i++) { + var path = paths[i]; + var name = path[0]; + var rest = path.slice(1); + var item = null; + for(var j = 0, m = items.length; j < m; j++) { + if(items[j].name === name) { + item = items[j]; + break; + } + } + if(item === null) { + item = {name: name, children: []}; + items.push(item); + } + if(rest.length > 0) { + item.children.push(rest); + } + } + for(i = 0, l = items.length; i < l; i++) { + item = items[i]; + item.children = structurise(item.children); + } + return items; +} + + +function stringify(items) { + var lines = []; + for(var i = 0, l = items.length; i < l; i++) { + var item = items[i]; + lines.push(item.name); + var subLines = stringify(item.children); + for(var j = 0, m = subLines.length; j < m; j++) { + lines.push(" " + subLines[j]); + } + } + return lines; +} diff --git a/magneticow/magneticow/static/styles/essential.css b/magneticow/magneticow/static/styles/essential.css new file mode 100644 index 0000000..6c940cb --- /dev/null +++ b/magneticow/magneticow/static/styles/essential.css @@ -0,0 +1,121 @@ +@font-face { + font-family: 'Noto Sans'; + src: url('../fonts/NotoSansUI/Regular.ttf'); + font-weight: normal; + font-style: normal; +} + +@font-face { + font-family: 'Noto Sans'; + src: url('../fonts/NotoSansUI/Bold.ttf'); + font-weight: bold; + font-style: normal; +} + +@font-face { + font-family: 'Noto Sans'; + src: url('../fonts/NotoSansUI/Italic.ttf'); + font-weight: normal; + font-style: italic; +} + +@font-face { + font-family: 'Noto Sans'; + src: url('../fonts/NotoSansUI/BoldItalic.ttf'); + font-weight: bold; + font-style: italic; +} + +@font-face { + font-family: 'Noto Mono'; + src: url('../fonts/NotoMono/Regular.ttf'); + font-weight: normal; + font-style: monospace; +} + + +html { + font-family: 'Noto Sans', sans-serif; +} + +pre { + font-family: 'Noto Mono'; + line-height: 1.2em; +} + +body { + padding: 1em 3em 1em 3em; + font-weight: 400; + line-height: 1.45; +} + +b { + font-weight: bold; +} + +/* Source: http://type-scale.com/ (using 1.200 - Minor Third Scale) */ +html { + font-size: 1em; +} + +p { + margin-bottom: 1.3em; +} + +h1, h2, h3, h4 { + margin: 1.414em 0 0.5em; + font-weight: inherit; + line-height: 1.2; +} + +h1 { + margin-top: 0; + font-size: 2.074em; +} + +h2 { + font-size: 1.728em; +} + +h3 { + font-size: 1.44em; +} + +h4 { + font-size: 1.2em; +} + +small { + font-size: 0.833em; +} + + +/* Source: https://gist.github.com/unruthless/413930 */ +sub, sup { + /* Specified in % so that the sup/sup is the + right size relative to the surrounding text */ + font-size: 75%; + + /* Zero out the line-height so that it doesn't + interfere with the positioning that follows */ + line-height: 0; + + /* Where the magic happens: makes all browsers position + the sup/sup properly, relative to the surrounding text */ + position: relative; + + /* Note that if you're using Eric Meyer's reset.css, this + is already set and you can remove this rule */ + vertical-align: baseline; +} + +sup { + /* Move the superscripted text up */ + top: -0.5em; +} + +sub { + /* Move the subscripted text down, but only + half as far down as the superscript moved up */ + bottom: -0.25em; +} \ No newline at end of file diff --git a/magneticow/magneticow/static/styles/homepage.css b/magneticow/magneticow/static/styles/homepage.css new file mode 100644 index 0000000..d6659e0 --- /dev/null +++ b/magneticow/magneticow/static/styles/homepage.css @@ -0,0 +1,23 @@ +main { + display: flex; + align-items: center; + align-content: center; + + height: calc(100vh - 2 * 3em); + width: calc(100vw - 2 * 3em); +} + +main form { + max-width: 600px; + width: 100%; + + margin-left: 0.5em; +} + +main form input { + width: 100%; +} + +main > div { + margin-right: 0.5em; +} \ No newline at end of file diff --git a/magneticow/magneticow/static/styles/reset.css b/magneticow/magneticow/static/styles/reset.css new file mode 100644 index 0000000..ed11813 --- /dev/null +++ b/magneticow/magneticow/static/styles/reset.css @@ -0,0 +1,48 @@ +/* http://meyerweb.com/eric/tools/css/reset/ + v2.0 | 20110126 + License: none (public domain) +*/ + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; +} +/* HTML5 display-role reset for older browsers */ +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} +body { + line-height: 1; +} +ol, ul { + list-style: none; +} +blockquote, q { + quotes: none; +} +blockquote:before, blockquote:after, +q:before, q:after { + content: ''; + content: none; +} +table { + border-collapse: collapse; + border-spacing: 0; +} diff --git a/magneticow/magneticow/static/styles/torrent.css b/magneticow/magneticow/static/styles/torrent.css new file mode 100644 index 0000000..8f57910 --- /dev/null +++ b/magneticow/magneticow/static/styles/torrent.css @@ -0,0 +1,74 @@ +header { + display: flex; + align-items: center; + align-content: center; + + width: 100%; + + padding-bottom: 0.833em; + border-bottom: 1px solid; + margin-bottom: 0.833em; +} + +header div a { + text-decoration: none; + color: inherit; +} + +header form { + max-width: 600px; + width: 100%; + + margin-left: 0.5em; +} + +header form input { + width: 100%; +} + +header > div { + margin-right: 0.5em; +} + +#title h2 { + margin-bottom: 0px; +} + +#title { + margin-bottom: 0.833em; +} + +#title a { + text-decoration: none; + color: inherit; +} + +pre { + background-color: black; + color: white; + overflow: auto; +} + +table { + max-width: 700px; + width: 700px; +} + +th { + padding-right: 0.833em; +} + +td { + padding-left: 0.833em; +} + +td, th { + white-space: nowrap; +} + +th { + font-weight: bold; + text-align: left; + + width: 1%; +} \ No newline at end of file diff --git a/magneticow/magneticow/static/styles/torrents.css b/magneticow/magneticow/static/styles/torrents.css new file mode 100644 index 0000000..45d42cd --- /dev/null +++ b/magneticow/magneticow/static/styles/torrents.css @@ -0,0 +1,142 @@ +header { + display: flex; + align-items: center; + align-content: center; + + width: 100%; + + padding-bottom: 0.833em; + border-bottom: 1px solid; + margin-bottom: 0.833em; +} + + +header div a { + text-decoration: none; + color: inherit; +} + + +header form { + max-width: 600px; + width: 100%; + + margin-left: 0.5em; +} + + +header form input { + width: 100%; +} + + +header > div { + margin-right: 0.5em; +} + +footer { + margin-top: 0.833em; + + display: flex; + justify-content: space-between; +} + + +footer form { + display: inline; +} + + +footer button { + font-size: 0.833em; +} + + +footer button:nth-child(1) { + margin-right: 0.833em; +} + + +footer button:nth-child(2) { + margin-left: 0.833em; +} + + +table { + width: 100%; +} + + +th { + font-weight: bold; + text-align: left; +} + + +tbody td, thead th { + padding-left: 0.833em; + padding-right: 0.833em; + + white-space: nowrap; +} + + +thead th:nth-child(1) { /* magnet link */ + width: 12px; +} + + +thead th:nth-child(2) { /* name */ + max-width: 0; + text-overflow: ellipsis; + overflow: hidden; +} + + +thead th:nth-child(3) { /* size */ + width: 75px; +} + + +tbody td:nth-child(3n+0) { + text-align: right; +} + + +thead th:nth-child(4) { /* discovered on */ + width: 120px; +} + +tbody tr:nth-child(2n+2) { + background-color: #efefef; +} + + +/* table in-borders */ +table { + border-collapse: collapse; +} + +table td, table th { + border-right: 1px solid black; +} + +table tr:first-child th { + border-top: 0; +} + +table tr:last-child td { + border-bottom: 0; +} + +table tr td:first-child, +table tr th:first-child { + border-left: 0; +} + +table tr td:last-child, +table tr th:last-child { + border-right: 0; +} + + diff --git a/magneticow/magneticow/templates/homepage.html b/magneticow/magneticow/templates/homepage.html new file mode 100644 index 0000000..3e0eed7 --- /dev/null +++ b/magneticow/magneticow/templates/homepage.html @@ -0,0 +1,19 @@ + + + + + magneticow + + + + + + +
+
magneticow​(pre-alpha)
+
+ +
+
+ + \ No newline at end of file diff --git a/magneticow/magneticow/templates/torrent.html b/magneticow/magneticow/templates/torrent.html new file mode 100644 index 0000000..d44271b --- /dev/null +++ b/magneticow/magneticow/templates/torrent.html @@ -0,0 +1,55 @@ + + + + + {{ torrent.name }} - magnetico + + + + + + +
+
magneticow​(pre-alpha)
+
+ +
+
+
+
+

{{ torrent.name }}

+ + Magnet link + {{ torrent.info_hash }} + +
+ + + + + + + + + + + + + + +
Size{{ torrent.size }}
Discovered on{{ torrent.discovered_on }}
Files{{ torrent.files|length }}
+ +

Contents

+ + + 
{% for file in torrent.files -%}{{ file.path }}{{ "\t" + file.size }}{{ "\n" }}{%- endfor %}
+
+ + \ No newline at end of file diff --git a/magneticow/magneticow/templates/torrents.html b/magneticow/magneticow/templates/torrents.html new file mode 100644 index 0000000..9b19ecf --- /dev/null +++ b/magneticow/magneticow/templates/torrents.html @@ -0,0 +1,56 @@ + + + + + {% if search %}"{{search}}"{% else %}Most recent torrents{% endif %} - magneticow + + + + + + +
+
magneticow​(pre-alpha)
+
+ +
+
+
+ + + + + + + + + + + {% for torrent in torrents %} + + + + + + + {% endfor %} + +
NameSizeDiscovered on
+ Magnet link{{ torrent.name }}{{ torrent.size }}{{ torrent.discovered_on }}
+
+
+ +
+ + + +
+
+ + + +
+
+ + \ No newline at end of file diff --git a/magneticow/magneticow/utils.py b/magneticow/magneticow/utils.py new file mode 100644 index 0000000..1297f6e --- /dev/null +++ b/magneticow/magneticow/utils.py @@ -0,0 +1,69 @@ +# magneticow - Lightweight web interface for magnetico. +# Copyright (C) 2017 Mert Bora ALPER +# Dedicated to Cemile Binay, in whose hands I thrived. +# +# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along with this program. If not, see +# . +from math import log10 +from struct import unpack_from + + +# Source: http://stackoverflow.com/a/1094933 +# (primarily: https://web.archive.org/web/20111010015624/http://blogmag.net/blog/read/38/Print_human_readable_file_size) +def to_human_size(num, suffix='B'): + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(num) < 1024: + return "%3.1f %s%s" % (num, unit, suffix) + num /= 1024 + return "%.1f %s%s" % (num, 'Yi', suffix) + + +def rank(blob): + # TODO: is there a way to futher optimize this? + p, c, n = unpack_from("=LLL", blob, 0) + + x = [] # list of tuples + for i in range(12, 12 + 3*c*p*4, 3*4): + x0, x1, x2 = unpack_from("=LLL", blob, i) + if x1 != 0: # skip if it's index column + x.append((x0, x1, x2)) + + # Ignore the first column (torrent_id) + _, avgdl = unpack_from("=LL", blob, 12 + 3*c*p*4) + + # Ignore the first column (torrent_id) + _, l = unpack_from("=LL", blob, (12 + 3*c*p*4) + 4*c) + + # Multiply by -1 so that sorting in the ASC order would yield the best match first + return -1 * okapi_bm25(term_frequencies=[X[0] for X in x], dl=l, avgdl=avgdl, N=n, nq=[X[2] for X in x]) + + +# TODO: check if I got it right =) +def okapi_bm25(term_frequencies, dl, avgdl, N, nq, k1=1.2, b=0.75): + """ + + :param term_frequencies: List of frequencies of each term in the document. + :param dl: Length of the document in words. + :param avgdl: Average document length in the collection. + :param N: Total number of documents in the collection. + :param nq: List of each numbers of documents containing term[i] for each term. + :param k1: Adjustable constant; = 1.2 in FTS5 extension of SQLite3. + :param b: Adjustable constant; = 0.75 in FTS5 extension of SQLite3. + :return: + """ + return sum( + log10((N - nq[i] + 0.5) / (nq[i] + 0.5)) * + ( + (term_frequencies[i] * (k1 + 1)) / + (term_frequencies[i] + k1 * (1 - b + b * dl / avgdl)) + ) + for i in range(len(term_frequencies)) + ) diff --git a/magneticow/setup.py b/magneticow/setup.py new file mode 100644 index 0000000..08627df --- /dev/null +++ b/magneticow/setup.py @@ -0,0 +1,41 @@ +from setuptools import setup + + +def read_file(path): + with open(path) as file: + return file.read() + + +setup( + name="magneticow", + version="0.1.0", + description="Lightweight web interface for magnetico.", + long_description=read_file("README.rst"), + url="http://magnetico.org", + author="Mert Bora ALPER", + author_email="bora@boramalper.org", + license="GNU Affero General Public License v3 or later (AGPLv3+)", + packages=["magneticow"], + include_package_data=True, + zip_safe=False, + entry_points={ + "console_scripts": ["magneticow=magneticow.__main__:main"] + }, + + install_requires=[ + "appdirs>=1.4.3", + "flask>=0.12.1", + "gevent>=1.2.1" + ], + + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Web Environment", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython" + ] +) diff --git a/magneticow/systemd/magneticow.service b/magneticow/systemd/magneticow.service new file mode 100644 index 0000000..f0870a8 --- /dev/null +++ b/magneticow/systemd/magneticow.service @@ -0,0 +1,10 @@ +[Unit] +Description=magneticow: lightweight web interface for magnetico + +[Service] +ExecStart=~/.local/bin/magneticow --port PORT --user USERNAME PASSWORD +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..14ddf8c --- /dev/null +++ b/pylintrc @@ -0,0 +1,407 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=.git + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=no + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence=INFERENCE + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=range-builtin-not-iterating,coerce-builtin,old-ne-operator,reduce-builtin,suppressed-message,parameter-unpacking,unichr-builtin,round-builtin,hex-method,dict-iter-method,basestring-builtin,no-absolute-import,using-cmp-argument,buffer-builtin,raw_input-builtin,delslice-method,filter-builtin-not-iterating,setslice-method,nonzero-method,import-star-module-level,useless-suppression,map-builtin-not-iterating,raising-string,file-builtin,dict-view-method,standarderror-builtin,long-suffix,print-statement,xrange-builtin,intern-builtin,input-builtin,metaclass-assignment,cmp-method,unpacking-in-except,cmp-builtin,next-method-called,coerce-method,apply-builtin,long-builtin,getslice-method,zip-builtin-not-iterating,backtick,execfile-builtin,unicode-builtin,old-division,indexing-exception,old-raise-syntax,oct-method,reload-builtin,old-octal-literal + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=colorized + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=120 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception