diff --git a/.gitignore b/.gitignore index 13d1490..a81c8ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -# ---> Python # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -21,7 +20,6 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -51,6 +49,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -73,6 +72,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -83,7 +83,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -129,3 +131,8 @@ dmypy.json # Pyre type checker .pyre/ +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/jd-distributed-crawler.iml b/.idea/jd-distributed-crawler.iml new file mode 100644 index 0000000..9a323b0 --- /dev/null +++ b/.idea/jd-distributed-crawler.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..dc9ea49 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..347624f --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/LICENSE b/LICENSE index e37e32e..dbbe355 100644 --- a/LICENSE +++ b/LICENSE @@ -1,613 +1,661 @@ -GNU AFFERO GENERAL PUBLIC LICENSE - -Version 3, 19 November 2007 - -Copyright (C) 2007 Free Software Foundation, Inc. - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The GNU Affero General Public License is a free, copyleft license for software -and other kinds of works, specifically designed to ensure cooperation with -the community in the case of network server software. - -The licenses for most software and other practical works are designed to take -away your freedom to share and change the works. By contrast, our General -Public Licenses are intended to guarantee your freedom to share and change -all versions of a program--to make sure it remains free software for all its -users. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom -to distribute copies of free software (and charge for them if you wish), that -you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs, and that you know you -can do these things. - -Developers that use our General Public Licenses protect your rights with two -steps: (1) assert copyright on the software, and (2) offer you this License -which gives you legal permission to copy, distribute and/or modify the software. - -A secondary benefit of defending all users' freedom is that improvements made -in alternate versions of the program, if they receive widespread use, become -available for other developers to incorporate. Many developers of free software -are heartened and encouraged by the resulting cooperation. However, in the -case of software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and letting -the public access it on a server without ever releasing its source code to -the public. - -The GNU Affero General Public License is designed specifically to ensure that, -in such cases, the modified source code becomes available to the community. -It requires the operator of a network server to provide the source code of -the modified version running there to the users of that server. Therefore, -public use of a modified version, on a publicly accessible server, gives the -public access to the source code of the modified version. - -An older license, called the Affero General Public License and published by -Affero, was designed to accomplish similar goals. This is a different license, -not a version of the Affero GPL, but Affero has released a new version of -the Affero GPL which permits relicensing under this license. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - -"Copyright" also means copyright-like laws that apply to other kinds of works, -such as semiconductor masks. - -"The Program" refers to any copyrightable work licensed under this License. -Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals -or organizations. - -To "modify" a work means to copy from or adapt all or part of the work in -a fashion requiring copyright permission, other than the making of an exact -copy. The resulting work is called a "modified version" of the earlier work -or a work "based on" the earlier work. - -A "covered work" means either the unmodified Program or a work based on the -Program. - -To "propagate" a work means to do anything with it that, without permission, -would make you directly or secondarily liable for infringement under applicable -copyright law, except executing it on a computer or modifying a private copy. -Propagation includes copying, distribution (with or without modification), -making available to the public, and in some countries other activities as -well. - -To "convey" a work means any kind of propagation that enables other parties -to make or receive copies. Mere interaction with a user through a computer -network, with no transfer of a copy, is not conveying. - -An interactive user interface displays "Appropriate Legal Notices" to the -extent that it includes a convenient and prominently visible feature that -(1) displays an appropriate copyright notice, and (2) tells the user that -there is no warranty for the work (except to the extent that warranties are -provided), that licensees may convey the work under this License, and how -to view a copy of this License. If the interface presents a list of user commands -or options, such as a menu, a prominent item in the list meets this criterion. - - 1. Source Code. - -The "source code" for a work means the preferred form of the work for making -modifications to it. "Object code" means any non-source form of a work. - -A "Standard Interface" means an interface that either is an official standard -defined by a recognized standards body, or, in the case of interfaces specified -for a particular programming language, one that is widely used among developers -working in that language. - -The "System Libraries" of an executable work include anything, other than -the work as a whole, that (a) is included in the normal form of packaging -a Major Component, but which is not part of that Major Component, and (b) -serves only to enable use of the work with that Major Component, or to implement -a Standard Interface for which an implementation is available to the public -in source code form. A "Major Component", in this context, means a major essential -component (kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to produce -the work, or an object code interpreter used to run it. - -The "Corresponding Source" for a work in object code form means all the source -code needed to generate, install, and (for an executable work) run the object -code and to modify the work, including scripts to control those activities. -However, it does not include the work's System Libraries, or general-purpose -tools or generally available free programs which are used unmodified in performing -those activities but which are not part of the work. For example, Corresponding -Source includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically linked -subprograms that the work is specifically designed to require, such as by -intimate data communication or control flow between those - - subprograms and other parts of the work. - -The Corresponding Source need not include anything that users can regenerate -automatically from other parts of the Corresponding Source. - - The Corresponding Source for a work in source code form is that same work. - - 2. Basic Permissions. - -All rights granted under this License are granted for the term of copyright -on the Program, and are irrevocable provided the stated conditions are met. -This License explicitly affirms your unlimited permission to run the unmodified -Program. The output from running a covered work is covered by this License -only if the output, given its content, constitutes a covered work. This License -acknowledges your rights of fair use or other equivalent, as provided by copyright -law. - -You may make, run and propagate covered works that you do not convey, without -conditions so long as your license otherwise remains in force. You may convey -covered works to others for the sole purpose of having them make modifications -exclusively for you, or provide you with facilities for running those works, -provided that you comply with the terms of this License in conveying all material -for which you do not control copyright. Those thus making or running the covered -works for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of your copyrighted -material outside their relationship with you. - -Conveying under any other circumstances is permitted solely under the conditions -stated below. Sublicensing is not allowed; section 10 makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - -No covered work shall be deemed part of an effective technological measure -under any applicable law fulfilling obligations under article 11 of the WIPO -copyright treaty adopted on 20 December 1996, or similar laws prohibiting -or restricting circumvention of such measures. - -When you convey a covered work, you waive any legal power to forbid circumvention -of technological measures to the extent such circumvention is effected by -exercising rights under this License with respect to the covered work, and -you disclaim any intention to limit operation or modification of the work -as a means of enforcing, against the work's users, your or third parties' -legal rights to forbid circumvention of technological measures. - - 4. Conveying Verbatim Copies. - -You may convey verbatim copies of the Program's source code as you receive -it, in any medium, provided that you conspicuously and appropriately publish -on each copy an appropriate copyright notice; keep intact all notices stating -that this License and any non-permissive terms added in accord with section -7 apply to the code; keep intact all notices of the absence of any warranty; -and give all recipients a copy of this License along with the Program. - -You may charge any price or no price for each copy that you convey, and you -may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - -You may convey a work based on the Program, or the modifications to produce -it from the Program, in the form of source code under the terms of section -4, provided that you also meet all of these conditions: - -a) The work must carry prominent notices stating that you modified it, and -giving a relevant date. - -b) The work must carry prominent notices stating that it is released under -this License and any conditions added under section 7. This requirement modifies -the requirement in section 4 to "keep intact all notices". - -c) You must license the entire work, as a whole, under this License to anyone -who comes into possession of a copy. This License will therefore apply, along -with any applicable section 7 additional terms, to the whole of the work, -and all its parts, regardless of how they are packaged. This License gives -no permission to license the work in any other way, but it does not invalidate -such permission if you have separately received it. - -d) If the work has interactive user interfaces, each must display Appropriate -Legal Notices; however, if the Program has interactive interfaces that do -not display Appropriate Legal Notices, your work need not make them do so. - -A compilation of a covered work with other separate and independent works, -which are not by their nature extensions of the covered work, and which are -not combined with it such as to form a larger program, in or on a volume of -a storage or distribution medium, is called an "aggregate" if the compilation -and its resulting copyright are not used to limit the access or legal rights -of the compilation's users beyond what the individual works permit. Inclusion -of a covered work in an aggregate does not cause this License to apply to -the other parts of the aggregate. - - 6. Conveying Non-Source Forms. - -You may convey a covered work in object code form under the terms of sections -4 and 5, provided that you also convey the machine-readable Corresponding -Source under the terms of this License, in one of these ways: - -a) Convey the object code in, or embodied in, a physical product (including -a physical distribution medium), accompanied by the Corresponding Source fixed -on a durable physical medium customarily used for software interchange. - -b) Convey the object code in, or embodied in, a physical product (including -a physical distribution medium), accompanied by a written offer, valid for -at least three years and valid for as long as you offer spare parts or customer -support for that product model, to give anyone who possesses the object code -either (1) a copy of the Corresponding Source for all the software in the -product that is covered by this License, on a durable physical medium customarily -used for software interchange, for a price no more than your reasonable cost -of physically performing this conveying of source, or (2) access to copy the -Corresponding Source from a network server at no charge. - -c) Convey individual copies of the object code with a copy of the written -offer to provide the Corresponding Source. This alternative is allowed only -occasionally and noncommercially, and only if you received the object code -with such an offer, in accord with subsection 6b. - -d) Convey the object code by offering access from a designated place (gratis -or for a charge), and offer equivalent access to the Corresponding Source -in the same way through the same place at no further charge. You need not -require recipients to copy the Corresponding Source along with the object -code. If the place to copy the object code is a network server, the Corresponding -Source may be on a different server (operated by you or a third party) that -supports equivalent copying facilities, provided you maintain clear directions -next to the object code saying where to find the Corresponding Source. Regardless -of what server hosts the Corresponding Source, you remain obligated to ensure -that it is available for as long as needed to satisfy these requirements. - -e) Convey the object code using peer-to-peer transmission, provided you inform -other peers where the object code and Corresponding Source of the work are -being offered to the general public at no charge under subsection 6d. - -A separable portion of the object code, whose source code is excluded from -the Corresponding Source as a System Library, need not be included in conveying -the object code work. - -A "User Product" is either (1) a "consumer product", which means any tangible -personal property which is normally used for personal, family, or household -purposes, or (2) anything designed or sold for incorporation into a dwelling. -In determining whether a product is a consumer product, doubtful cases shall -be resolved in favor of coverage. For a particular product received by a particular -user, "normally used" refers to a typical or common use of that class of product, -regardless of the status of the particular user or of the way in which the -particular user actually uses, or expects or is expected to use, the product. -A product is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent the -only significant mode of use of the product. - -"Installation Information" for a User Product means any methods, procedures, -authorization keys, or other information required to install and execute modified -versions of a covered work in that User Product from a modified version of -its Corresponding Source. The information must suffice to ensure that the -continued functioning of the modified object code is in no case prevented -or interfered with solely because modification has been made. - -If you convey an object code work under this section in, or with, or specifically -for use in, a User Product, and the conveying occurs as part of a transaction -in which the right of possession and use of the User Product is transferred -to the recipient in perpetuity or for a fixed term (regardless of how the -transaction is characterized), the Corresponding Source conveyed under this -section must be accompanied by the Installation Information. But this requirement -does not apply if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has been installed -in ROM). - -The requirement to provide Installation Information does not include a requirement -to continue to provide support service, warranty, or updates for a work that -has been modified or installed by the recipient, or for the User Product in -which it has been modified or installed. Access to a network may be denied -when the modification itself materially and adversely affects the operation -of the network or violates the rules and protocols for communication across -the network. - -Corresponding Source conveyed, and Installation Information provided, in accord -with this section must be in a format that is publicly documented (and with -an implementation available to the public in source code form), and must require -no special password or key for unpacking, reading or copying. - - 7. Additional Terms. - -"Additional permissions" are terms that supplement the terms of this License -by making exceptions from one or more of its conditions. Additional permissions -that are applicable to the entire Program shall be treated as though they -were included in this License, to the extent that they are valid under applicable -law. If additional permissions apply only to part of the Program, that part -may be used separately under those permissions, but the entire Program remains -governed by this License without regard to the additional permissions. - -When you convey a copy of a covered work, you may at your option remove any -additional permissions from that copy, or from any part of it. (Additional -permissions may be written to require their own removal in certain cases when -you modify the work.) You may place additional permissions on material, added -by you to a covered work, for which you have or can give appropriate copyright -permission. - -Notwithstanding any other provision of this License, for material you add -to a covered work, you may (if authorized by the copyright holders of that -material) supplement the terms of this License with terms: - -a) Disclaiming warranty or limiting liability differently from the terms of -sections 15 and 16 of this License; or - -b) Requiring preservation of specified reasonable legal notices or author -attributions in that material or in the Appropriate Legal Notices displayed -by works containing it; or - -c) Prohibiting misrepresentation of the origin of that material, or requiring -that modified versions of such material be marked in reasonable ways as different -from the original version; or - -d) Limiting the use for publicity purposes of names of licensors or authors -of the material; or - -e) Declining to grant rights under trademark law for use of some trade names, -trademarks, or service marks; or - -f) Requiring indemnification of licensors and authors of that material by -anyone who conveys the material (or modified versions of it) with contractual -assumptions of liability to the recipient, for any liability that these contractual -assumptions directly impose on those licensors and authors. - -All other non-permissive additional terms are considered "further restrictions" -within the meaning of section 10. If the Program as you received it, or any -part of it, contains a notice stating that it is governed by this License -along with a term that is a further restriction, you may remove that term. -If a license document contains a further restriction but permits relicensing -or conveying under this License, you may add to a covered work material governed -by the terms of that license document, provided that the further restriction -does not survive such relicensing or conveying. - -If you add terms to a covered work in accord with this section, you must place, -in the relevant source files, a statement of the additional terms that apply -to those files, or a notice indicating where to find the applicable terms. - -Additional terms, permissive or non-permissive, may be stated in the form -of a separately written license, or stated as exceptions; the above requirements -apply either way. - - 8. Termination. - -You may not propagate or modify a covered work except as expressly provided -under this License. Any attempt otherwise to propagate or modify it is void, -and will automatically terminate your rights under this License (including -any patent licenses granted under the third paragraph of section 11). - -However, if you cease all violation of this License, then your license from -a particular copyright holder is reinstated (a) provisionally, unless and -until the copyright holder explicitly and finally terminates your license, -and (b) permanently, if the copyright holder fails to notify you of the violation -by some reasonable means prior to 60 days after the cessation. - -Moreover, your license from a particular copyright holder is reinstated permanently -if the copyright holder notifies you of the violation by some reasonable means, -this is the first time you have received notice of violation of this License -(for any work) from that copyright holder, and you cure the violation prior -to 30 days after your receipt of the notice. - -Termination of your rights under this section does not terminate the licenses -of parties who have received copies or rights from you under this License. -If your rights have been terminated and not permanently reinstated, you do -not qualify to receive new licenses for the same material under section 10. - - 9. Acceptance Not Required for Having Copies. - -You are not required to accept this License in order to receive or run a copy -of the Program. Ancillary propagation of a covered work occurring solely as -a consequence of using peer-to-peer transmission to receive a copy likewise -does not require acceptance. However, nothing other than this License grants -you permission to propagate or modify any covered work. These actions infringe -copyright if you do not accept this License. Therefore, by modifying or propagating -a covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - -Each time you convey a covered work, the recipient automatically receives -a license from the original licensors, to run, modify and propagate that work, -subject to this License. You are not responsible for enforcing compliance -by third parties with this License. - -An "entity transaction" is a transaction transferring control of an organization, -or substantially all assets of one, or subdividing an organization, or merging -organizations. If propagation of a covered work results from an entity transaction, -each party to that transaction who receives a copy of the work also receives -whatever licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the Corresponding -Source of the work from the predecessor in interest, if the predecessor has -it or can get it with reasonable efforts. - -You may not impose any further restrictions on the exercise of the rights -granted or affirmed under this License. For example, you may not impose a -license fee, royalty, or other charge for exercise of rights granted under -this License, and you may not initiate litigation (including a cross-claim -or counterclaim in a lawsuit) alleging that any patent claim is infringed -by making, using, selling, offering for sale, or importing the Program or -any portion of it. - - 11. Patents. - -A "contributor" is a copyright holder who authorizes use under this License -of the Program or a work on which the Program is based. The work thus licensed -is called the contributor's "contributor version". - -A contributor's "essential patent claims" are all patent claims owned or controlled -by the contributor, whether already acquired or hereafter acquired, that would -be infringed by some manner, permitted by this License, of making, using, -or selling its contributor version, but do not include claims that would be -infringed only as a consequence of further modification of the contributor -version. For purposes of this definition, "control" includes the right to -grant patent sublicenses in a manner consistent with the requirements of this -License. - -Each contributor grants you a non-exclusive, worldwide, royalty-free patent -license under the contributor's essential patent claims, to make, use, sell, -offer for sale, import and otherwise run, modify and propagate the contents -of its contributor version. - -In the following three paragraphs, a "patent license" is any express agreement -or commitment, however denominated, not to enforce a patent (such as an express -permission to practice a patent or covenant not to s ue for patent infringement). -To "grant" such a patent license to a party means to make such an agreement -or commitment not to enforce a patent against the party. - -If you convey a covered work, knowingly relying on a patent license, and the -Corresponding Source of the work is not available for anyone to copy, free -of charge and under the terms of this License, through a publicly available -network server or other readily accessible means, then you must either (1) -cause the Corresponding Source to be so available, or (2) arrange to deprive -yourself of the benefit of the patent license for this particular work, or -(3) arrange, in a manner consistent with the requirements of this License, -to extend the patent - -license to downstream recipients. "Knowingly relying" means you have actual -knowledge that, but for the patent license, your conveying the covered work -in a country, or your recipient's use of the covered work in a country, would -infringe one or more identifiable patents in that country that you have reason -to believe are valid. - -If, pursuant to or in connection with a single transaction or arrangement, -you convey, or propagate by procuring conveyance of, a covered work, and grant -a patent license to some of the parties receiving the covered work authorizing -them to use, propagate, modify or convey a specific copy of the covered work, -then the patent license you grant is automatically extended to all recipients -of the covered work and works based on it. - -A patent license is "discriminatory" if it does not include within the scope -of its coverage, prohibits the exercise of, or is conditioned on the non-exercise -of one or more of the rights that are specifically granted under this License. -You may not convey a covered work if you are a party to an arrangement with -a third party that is in the business of distributing software, under which -you make payment to the third party based on the extent of your activity of -conveying the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory patent -license (a) in connection with copies of the covered work conveyed by you -(or copies made from those copies), or (b) primarily for and in connection -with specific products or compilations that contain the covered work, unless -you entered into that arrangement, or that patent license was granted, prior -to 28 March 2007. - -Nothing in this License shall be construed as excluding or limiting any implied -license or other defenses to infringement that may otherwise be available -to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - -If conditions are imposed on you (whether by court order, agreement or otherwise) -that contradict the conditions of this License, they do not excuse you from -the conditions of this License. If you cannot convey a covered work so as -to satisfy simultaneously your obligations under this License and any other -pertinent obligations, then as a consequence you may - -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey the -Program, the only way you could satisfy both those terms and this License -would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - -Notwithstanding any other provision of this License, if you modify the Program, -your modified version must prominently offer all users interacting with it -remotely through a computer network (if your version supports such interaction) -an opportunity to receive the Corresponding Source of your version by providing -access to the Corresponding Source from a network server at no charge, through -some standard or customary means of facilitating copying of software. This -Corresponding Source shall include the Corresponding Source for any work covered -by version 3 of the GNU General Public License that is incorporated pursuant -to the following paragraph. - -Notwithstanding any other provision of this License, you have permission to -link or combine any covered work with a work licensed under version 3 of the -GNU General Public License into a single combined work, and to convey the -resulting work. The terms of this License will continue to apply to the part -which is the covered work, but the work with which it is combined will remain -governed by version 3 of the GNU General Public License. - - 14. Revised Versions of this License. - -The Free Software Foundation may publish revised and/or new versions of the -GNU Affero General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to address -new problems or concerns. - -Each version is given a distinguishing version number. If the Program specifies -that a certain numbered version of the GNU Affero General Public License "or -any later version" applies to it, you have the option of following the terms -and conditions either of that numbered version or of any later version published -by the Free Software Foundation. If the Program does not specify a version -number of the GNU Affero General Public License, you may choose any version -ever published by the Free Software Foundation. - -If the Program specifies that a proxy can decide which future versions of -the GNU Affero General Public License can be used, that proxy's public statement -of acceptance of a version permanently authorizes you to choose that version -for the Program. - -Later license versions may give you additional or different permissions. However, -no additional obligations are imposed on any author or copyright holder as -a result of your choosing to follow a later version. - - 15. Disclaimer of Warranty. - -THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE -LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER -EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM -PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR -CORRECTION. - - 16. Limitation of Liability. - -IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM -AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, -INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO -USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED -INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE -PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER -PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - -If the disclaimer of warranty and limitation of liability provided above cannot -be given local legal effect according to their terms, reviewing courts shall -apply local law that most closely approximates an absolute waiver of all civil -liability in connection with the Program, unless a warranty or assumption -of liability accompanies a copy of the Program in return for a fee. END OF -TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively state the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - - -Copyright (C) - -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU Affero General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -details. - -You should have received a copy of the GNU Affero General Public License along -with this program. If not, see . + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. -If your software can interact with users remotely through a computer network, -you should also make sure that it provides a way for users to get its source. -For example, if your program is a web application, its interface could display -a "Source" link that leads users to an archive of the code. There are many -ways you could offer source, and different solutions will be better for different -programs; see section 13 for the specific requirements. - -You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. For -more information on this, and how to apply and follow the GNU AGPL, see . + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..00e91ac --- /dev/null +++ b/README.en.md @@ -0,0 +1,37 @@ +# 京东分布式爬虫 + +#### Description +selenium爬取搜索页 +scrapy爬取详情页 + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/README.md b/README.md index 021bf44..00b77dc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,221 @@ -# jd-distributed-crawler +# 京东显卡商品信息分布式爬虫 +### 介绍 + +本爬虫分两个部分: + +- 第一部分搜索页爬虫是使用selenium爬取搜索页, 主要获取的是商品的sku和价格 + +- 第二部分商品页爬虫使用scrapy爬取详情页的商品介绍信息, 主要获取商品的型号 + +由于京东商品种类繁多, 不同的商品品类间差异巨大, 商品页爬虫的代码也就不近相同. 出于成本和时间考虑, 搜索页爬虫使用**3080**作为搜索关键词, 商品页爬虫爬取搜索页爬虫获取的商品对应的详情页 + +其整体架构如下: + +![](D:\python_class\jd-distributed-crawler\img\2022-04-15-16-28-10-image.png) + +### 电商网站筛选 + +爬虫的目的就是为了获取一定量的有价值的信息, 就目前中文互联网来说, 电商网站是信息密度和价值密度比较高的地方, 因此使用爬虫获取信息有一定的可用性. + +目前而言, 淘宝获取的商品信息结果层次最多, 品类最全. 但是首先淘宝商品搜索页混杂天猫,天猫国际, 淘宝卖家等各类各不相同的店铺, 因此其商品详情页结构也大相径庭; 其次是淘宝历史包袱比较重, 因此有许多结构混乱的页面混杂其中, 难以提取数据. + +因此, 实际考虑后确认, 淘宝不是太适合新人小团队去学习试炼爬虫爬取信息. + +相对而言, 京东搜索页基本同一为京东自己的页面, 京东自营, 旗舰店和商家店结构基本没有区别, 详情页结构也一致, 对爬虫比较友好(京东唯一不太友好的地方只有, 其主站页面下并没有写robots.txt). 因此考虑爬取京东. + +### 搜索页爬虫 + +主要由`settings.yaml`和`jdSearchSeleniumSpider.py`两部分组成,`settings.yaml`是配置文件, `jdSearchSeleniumSpider.py`是代码执行主体 + +#### 前期工作 + +基本而言, 任何有搜索功能的网站都会对其搜索功能做一定的限制, 以防止单个ip过高频率的访问, 占用过多资源. + +京东也不例外, 对于未登录的请求, 甚至是带着header和cookie的请求, 往往都是秉持拒绝的态度, 要求验证登录: + +![](D:\python_class\jd-distributed-crawler\img\2022-04-15-13-25-48-image.png) + +![](D:\python_class\jd-distributed-crawler\img\2022-04-15-13-26-42-image.png) + +搜索引擎的爬虫验证一般级别都比较高, 难以绕过. 考虑到搜索页的数量相对于商品页的数量往往是少一个数量级, 且京东单个搜索词的页面量也大致在几十几百页的数量级之间, 因此使用python的`selenium`库, 驱动浏览器去自动化访问搜索页, 进而获取数据, 是比较简单且可行的方案. + +#### settings.yaml + +`setting.yaml`是搜索页爬虫的配置文件, 其内容如下: + +```yaml +search: + keyword: '3080' + sleep: 5 + +redis: + host: '120.24.87.40' + port: '6379' + password: 'Guet@207' + db: 1 + +mongodb: + host: '120.24.87.40' + port: '27017' + username: "admin" + password: "123456" + +result: + rpush_key: "jd:start_urls" +``` + +- `search` + + - `keyword`: 要搜索的关键字, 即在京东商品搜索上的要搜索的商品; + + - `sleep`: 休眠时间, 单位: 秒; 由于**selenium**操作页面的动作(例如点击翻页, 下拉)是**异步**的, 而读取页面是**同步**的. 京东需要执行下拉这个操作后才加载出所有的商品信息, 为了读取全当前的页面信息, 需要休眠一段时间等待异步操作完成; + +- `redis` + + - `host`: redis部署的服务器地址; + + - `port`: redis部署时开放的端口号; + + - `password`: redis部署时设置的auth; + + - `db`: 需要使用redis的第几个数据库 + +- `mongodb` + + - `host`: MongoDB部署的服务器地址; + + - `port`: MongoDB部署时开放的端口号; + + - `username`: MongoDB部署时设置的auth的用户名, 或者说要使用的MongoDB的数据库名; + + - `password`: MongoDB部署时设置的auth. + +- `result` + + - `rpush_key`: 爬虫获取到的数据形成的url要放入redis时设置的key, url在Redis中存储方式是列表 + +#### jdSearchSeleniumSpider.py + +搜索页爬虫的程序入口, 根据京东搜索页面爬取页面元素. + +启动程序后进入京东的商品搜索页面. 搜索页的搜索关键字为`settings.py`中设定的`search.keyword`. + +京东的搜索页面会分两个阶段加载, 第一个加载阶段加载成功后(即加载出30个商品的信息), 爬虫会执行`scroll_by_xpath()`, 作用是滚动网页到网页底部, 触发京东搜索页面的第二个加载阶段, 把一个页面上的信息都加载到浏览器里. + +`scroll_by_xpath()` 内部原理就是驱动浏览器移动视图, 使视图可以看到指定的元素. 在该爬虫中, 指定的网页元素的**xpath**为`//*[@id="J_bottomPage"]/span[1]/a[9]`, 代表的是京东搜索页面的下一页按钮这一网页元素. + +两个阶段都加载完成后, 获取网页中所有商品的sku和价格. + +![](D:\python_class\jd-distributed-crawler\img\2022-04-15-18-40-23-image.png) + +sku是京东给商品的编号, 且可以根据sku拼接对应商品的详情页的URL. + +获取到sku和价格后, 将sku拼接为URL, rpush到Redis设置好的数据库的`result.rpush_key`中, 并且将每个商品的数据以`{'sku': sku, 'price': price}`的形式, 放入MongoDB设置好的数据库中. 放入MongoDB前会根据sku字段查询数据是否存在(原子操作, 可分布式), 防止重复插入. + +### 商品页爬虫 + +商品页爬虫使用了`scrapy_redis`去写, 即`Scrapy`的redis版本, 他与`Scrapy`的主要区别就是把存放URL于Reids数据库中, 利用Redis的特性, 即在多个请求请求同一个数据时, 会默认对请求排序, 有一个队列效果, 保证了分布式爬虫获取到不同的URL. + +`scrapy_redis`框架下的爬虫执行是异步的, 即, 从Redis数据库的列表中获取URL后, 发起Request, 便可以执行下一次从Redis中获取Redis, 而发起Request得到的response则等得到后再由回调函数处理(即执行`jdsku.py`里的`parse`方法). + +该爬虫只需要在**本爬虫目录下**, 调用`scrapy crawl jdsku`即可启动. + +#### settings.py + +配置文件, 具体配置查看`Scrapy`的配置, 其中有几个是`scrapy_redis`的配置或本次项目引入的配置, 在此说明: + + + +个人配置: + +`PROXY_SERVER_URL`: IP代理商提供的直连IP的API地址; + +`TEST_PAGE`: 用于测试IP代理商提供的IP代理服务是否能访问JD的测试地址, 可以按需更换不同的京东商品地址; + +`mongodb`: 配置时需要以字典形式配置 + +- `host`: MongoDB服务器的地址; + +- `port`: MongoDB服务器暴露的端口; + +- `username`: MongoDB要使用的用户名(也是数据库名) + +- `password`: MongoDB对应用户名的验证密码 + +`cookies`: 浏览器访问京东时的`cookies`, 有助于模拟浏览器访问, 降低失败率 + + + +scrapy_redis配置: + +`DONT_FILTER`: Redis访问不去重, True代表不去重. 不去重的原因是使用代理访问商品详情页时可能会失败, 此时会将URL重新放回Redis的列表开头; + +`SCHEDULER='scrapy_redis.scheduler.Scheduler'`: 使用scrapy_redis中的调度器, 即保证每台主机爬取的URL地址都不同的Scheduler. + +`DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter'`: 配置scrapy使用的去重类, 即RFPDupeFilter. + + `SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat"`: 使用`scrapy_redis.picklecompat`作为序列化, 不序列化的数据可能会出现乱码. + +   `AUTOTHROTTLE_ENABLED = True`: 开启自动限速, 控制爬取速度，降低对方服务器压力. + +#### items.py + +代码如下: + +```python +import scrapy + + +class JdskuspiderItem(scrapy.Item): + # sku-id + sku = scrapy.Field() + # title + title = scrapy.Field() + # 显卡型号 + model = scrapy.Field() + # 显存类型 + memoryType = scrapy.Field() + # 显存位宽 + bitWidth = scrapy.Field() + # 显存容量 + memoryCapacity = scrapy.Field() +``` + +item是爬虫数据的容器, 方便我们按对应字段存储数据 + + + +#### jdsku.py + +商品页爬虫的程序入口, 根据redis数据库对应的列表访问对应URL, 在京东商品详情页面爬取页面元素. 因为使用`scrapy_redis`框架, 也就是`Scrapy`的Redis分布式版本, 因此大部分代码都是在重写方法. + +对于`class JdskuSpider(RedisSpider)`: + +**属性**: + +`name`: 爬虫的名字, 后续在终端启动爬虫时需要输入的爬虫名; + +`allowed_domains`: 运行爬取的URL范围; + +`redis_key`: Redis数据库中存放要爬取的URL的列表的Key名, 会获取列表中的第一个数据; + +`redis_connection`: 存放一个新的Redis连接, 该连接后续用于访问失败时重新把URL返回给Redis列表; + +`mongo_connection`: 存放一个新的MongoDB连接, 该连接后续用于存放最终数据. + +**方法**: + +两个方法都是重写方法, 方法名和参数都固定, 两个方法都是自动调用. + +`make_requests_from_url(self, url)`: + +- `scrapy_redis`框架会调用该方法, 发起一个访问参数里URL的request, 返回一个response, 框架会将response发给他的回调函数处理(即下述的`parse`); + +- `meta={'url':url}`表示会将`{'url': url}`作为数据传递下去, 这样在他的回调函数(`parse`)里就能使用到该数据. + +`parse(self, response)`: + +- `scrapy_redis`发送的请求得到response后的回调函数; + +- 作用是解析商品详情页(针对特定商品), 提取数据, 并将其保存到MongoDB中, 若提取数据表现出来的是被京东拦截, 那么把该次访问的URL重新返回到Redis列表开头. diff --git a/img/2022-04-15-13-25-48-image.png b/img/2022-04-15-13-25-48-image.png new file mode 100644 index 0000000..5925370 Binary files /dev/null and b/img/2022-04-15-13-25-48-image.png differ diff --git a/img/2022-04-15-13-26-38-image.png b/img/2022-04-15-13-26-38-image.png new file mode 100644 index 0000000..c23afb4 Binary files /dev/null and b/img/2022-04-15-13-26-38-image.png differ diff --git a/img/2022-04-15-13-26-42-image.png b/img/2022-04-15-13-26-42-image.png new file mode 100644 index 0000000..c23afb4 Binary files /dev/null and b/img/2022-04-15-13-26-42-image.png differ diff --git a/img/2022-04-15-16-28-10-image.png b/img/2022-04-15-16-28-10-image.png new file mode 100644 index 0000000..5158e41 Binary files /dev/null and b/img/2022-04-15-16-28-10-image.png differ diff --git a/img/2022-04-15-18-40-20-image.png b/img/2022-04-15-18-40-20-image.png new file mode 100644 index 0000000..9f58866 Binary files /dev/null and b/img/2022-04-15-18-40-20-image.png differ diff --git a/img/2022-04-15-18-40-23-image.png b/img/2022-04-15-18-40-23-image.png new file mode 100644 index 0000000..9f58866 Binary files /dev/null and b/img/2022-04-15-18-40-23-image.png differ diff --git a/jdSearchSpider/jdSearchSeleniumSpider.py b/jdSearchSpider/jdSearchSeleniumSpider.py new file mode 100644 index 0000000..09ebcb6 --- /dev/null +++ b/jdSearchSpider/jdSearchSeleniumSpider.py @@ -0,0 +1,115 @@ +from time import sleep +import yaml +import redis +from selenium import webdriver +from selenium.webdriver.common.by import By +from pymongo import MongoClient + + +def scroll_by_xpath(webDriver, xpath): + """ + 将webDriver里的网页下拉到xpath的位置 + :param webDriver: web driver, 用于操作驱动后的浏览器 + :param xpath: 要定位的元素的xpath, str + """ + target = webDriver.find_element(By.XPATH, xpath) + webDriver.execute_script("arguments[0].scrollIntoView();", target) + + +if __name__ == '__main__': + # 获取配置文件 + yaml_file = "./settings.yaml" + with open(yaml_file, 'r') as f: + settings = yaml.safe_load(f) + + # 获取redis连接 + redis_connection = redis.Redis( + host=settings['redis']['host'], + port=settings['redis']['port'], + password=settings['redis']['password'], + db=settings['redis']['db'], + charset='UTF-8', + encoding='UTF-8') + + mongo_connection = MongoClient('mongodb://{}:{}'.format(settings['mongodb']['host'], settings['mongodb']['port']), + username = settings['mongodb']['username'], + password = settings['mongodb']['password']) + + # 打开京东商品搜索页 + driver = webdriver.Chrome() + driver.get('https://search.jd.com/Search?keyword={}&enc=utf-8&wq={}'.format(settings['search']['keyword'], settings['search']['keyword'])) + # 下拉到翻页导航的位置 + scroll_by_xpath(driver, '//*[@id="J_bottomPage"]/span[1]/a[9]') + # 强制休眠5s再读取元素, 京东的搜索页需要下拉才加载全, 而下拉操作是异步的, 不匹配下拉时间读取不全元素 + sleep(settings['search']['sleep']) + # 获取当前搜索的关键词的最大商品页 + max_page = int(driver.find_element(By.XPATH, '//*[@id="J_bottomPage"]/span[2]/em[1]/b').text) + + skuid_prev = [] + skuid_url = [] + page = 0 + while page < max_page: + sku_price = [] + + # 下拉 + scroll_by_xpath(driver, '//*[@id="J_bottomPage"]/span[1]/a[9]') + sleep(settings['search']['sleep']) + + # 获取搜索页的商品的skuid和对应的价格 + li_list = driver.find_elements(By.XPATH, '//*[@id="J_goodsList"]/ul/li') + for li in li_list: + li_text = li.text.split("\n") + result = 0 + for text in li_text: + if text[1] in [str(i) for i in range(10)] and text[2] in [str(i) for i in range(10)]: + result = text[1:-1] + break + price = result + sku_price.append({li.get_attribute("data-sku"): price}) + # 重复读取时, 重来 + if page != 0 and sku_price[0] == skuid_prev[0]: + page -= 1 + continue + # 防止未加载完就读取元素, 重新读取 + while len(skuid_prev) < len(skuid_prev): + li_list = driver.find_elements(By.XPATH, '//*[@id="J_goodsList"]/ul/li') + for li in li_list: + price = li.find_element(By.XPATH, "//*[@class='gl-i-wrap']/*[@class='p-price']//strong/i") + sku_price.append({li.get_attribute("data-sku"): price}) + + #拼接成url, 入redis的start_url + for dic in sku_price: + for key in dic.keys(): + skuid_url.append('https://item.jd.com/{}.html'.format(key)) + for url in skuid_url: + redis_connection.rpush(settings['result']['rpush_key'], url) + + #将sku和price放入mongodb + mongoDB = mongo_connection['admin'] + mongoCollection = mongoDB['a'] + + for dic in sku_price: + document = {} + for key in dic.keys(): + document['sku'] = key + document['price'] = dic[key] + # 更新MongoDB: + # 当数据库里没有, 插入 + mongoCollection.update_one( + {'sku': document['sku']}, + { + '$setOnInsert': document #生效需要upsert=True + }, + upsert=True + ) + + print(str(page) + " " + str(len(skuid_url)) + " ", end="") + print(sku_price) + print(skuid_url) + sleep(settings['search']['sleep']) + + skuid_prev = sku_price.copy() + skuid_url = [] + page += 1 + # 翻页 + driver.find_element(By.XPATH, '//*[@id="J_bottomPage"]/span[1]/a[9]').click() \ No newline at end of file diff --git a/jdSearchSpider/settings.yaml b/jdSearchSpider/settings.yaml new file mode 100644 index 0000000..a6bbc73 --- /dev/null +++ b/jdSearchSpider/settings.yaml @@ -0,0 +1,18 @@ +search: + keyword: '3080' + sleep: 5 + +redis: + host: '120.24.87.40' + port: '6379' + password: 'Guet@207' + db: 1 + +mongodb: + host: '120.24.87.40' + port: '27017' + username: "admin" + password: "123456" + +result: + rpush_key: "jd:start_urls" \ No newline at end of file diff --git a/jdSkuSpider/jdSkuSpider/__init__.py b/jdSkuSpider/jdSkuSpider/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jdSkuSpider/jdSkuSpider/items.py b/jdSkuSpider/jdSkuSpider/items.py new file mode 100644 index 0000000..b3741bf --- /dev/null +++ b/jdSkuSpider/jdSkuSpider/items.py @@ -0,0 +1,28 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class JdskuspiderItem(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + # sku-id + sku = scrapy.Field() + + # title + title = scrapy.Field() + + # 显卡型号 + model = scrapy.Field() + + # 显存类型 + memoryType = scrapy.Field() + + # 显存位宽 + bitWidth = scrapy.Field() + + # 显存容量 + memoryCapacity = scrapy.Field() diff --git a/jdSkuSpider/jdSkuSpider/middlewares.py b/jdSkuSpider/jdSkuSpider/middlewares.py new file mode 100644 index 0000000..21a80f6 --- /dev/null +++ b/jdSkuSpider/jdSkuSpider/middlewares.py @@ -0,0 +1,177 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html +import random + +from scrapy import signals +import requests +import logging + +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + + +class JdskuspiderSpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class JdskuspiderDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + +class ProxyMiddleWare: + + logger = logging.getLogger(__name__) + proxy = '' + + def get_random_proxy(self, spider): + """ + 从代理页面获取代理url + :return: 代理url + """ + try: + response = requests.get(spider.settings.get('PROXY_SERVER_URL')) + if response.status_code == 200: + # 得到类似['125.111.146.122:4345', '114.239.146.151:4356', ''] + url_list = response.text.split("\r\n") + # 去掉最后一个空字符串 + url_list.pop(-1) + + proxy = '{}'.format(random.choice(url_list)) + ip = {"http": "http://" + proxy} + logging.info("=====获取代理: {}=====".format(ip)) + + r = requests.get("http://www.baidu.com", proxies=ip, timeout=10) + if r.status_code == 200: + logging.info("=====代理通过测试, 选择该ip: {}=====".format(ip)) + return proxy + except: + logging.info('=====未通过代理测试, 重新从代理商处获取代理=====') + return self.get_random_proxy(spider) + + def get_proxy(self, spider): + """ + 从代理池里获取代理, 节省代理个数 + :return: proxy, 形式为: '125.111.146.122:4345' + """ + if self.proxy == "": + self.proxy = self.get_random_proxy(spider) + else: + ip = {"http": "http://" + str(self.proxy)} + r = requests.get(spider.settings.get("TEST_PAGE"), proxies=ip, timeout=10) + if r.status_code == 200: + #'window.location.href'等等...字段是封ip页面开头有的元素 + if not "window.location.href" in r.text[0:50]: + logging.info("=====代理通过测试, 继续使用该ip: {}=====".format(self.proxy)) + else: + self.proxy = self.get_random_proxy(spider) + else: + self.proxy = self.get_random_proxy(spider) + return self.proxy + + def process_request(self, request, spider): + """ + scrapy.middlewares模板的方法 + 使用代理对要爬取的网页request + """ + proxy = self.get_proxy(spider) + if proxy: + self.logger.debug('======' + '使用代理 ' + str(proxy) + "======") + request.meta['proxy'] = 'https://{proxy}'.format(proxy=proxy) + + def process_response(self, request, response, spider): + """ + scrapy.middlewares模板的方法 + 使用代理对要爬取的网页request失败时, 重新使用代理爬取 + """ + if response.status != 200: + self.logger.info("=====使用代理ip request网页获得失败response, 重新获取代理ip=====") + request.meta['proxy'] = 'https://{proxy}'.format(proxy=self.get_random_proxy(spider)) + return request + return response \ No newline at end of file diff --git a/jdSkuSpider/jdSkuSpider/pipelines.py b/jdSkuSpider/jdSkuSpider/pipelines.py new file mode 100644 index 0000000..e79da1b --- /dev/null +++ b/jdSkuSpider/jdSkuSpider/pipelines.py @@ -0,0 +1,14 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter + + +class JdskuspiderPipeline: + def process_item(self, item, spider): + print(item) + return item diff --git a/jdSkuSpider/jdSkuSpider/settings.py b/jdSkuSpider/jdSkuSpider/settings.py new file mode 100644 index 0000000..9907e3c --- /dev/null +++ b/jdSkuSpider/jdSkuSpider/settings.py @@ -0,0 +1,160 @@ +# Scrapy settings for jdSkuSpider project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'jdSkuSpider' + +SPIDER_MODULES = ['jdSkuSpider.spiders'] +NEWSPIDER_MODULE = 'jdSkuSpider.spiders' + +#代理商提供的代理网址 +PROXY_SERVER_URL = 'https://h.shanchendaili.com/api.html?action=get_ip&key=HU015c86520413222339Rp2a&time=10&count=1&protocol=http&type=text&textSep=1&only=1' +#用来测试当前的代理ip是否被封禁 +TEST_PAGE = 'https://item.jd.com/100017846659.html' +#不去重(True代表不去重, False代表去重) +DONT_FILTER = True + +REDIS_HOST = '120.24.87.40' +REDIS_PORT = '6379' +REDIS_DB = 1 +REDIS_PARAMS = { + 'password': 'Guet@207', +} +# 使用scrapy_redis中的调度器, 即保证每台主机爬取的URL地址都不同的Scheduler +SCHEDULER = 'scrapy_redis.scheduler.Scheduler' +# 配置scrapy使用的去重类, 即RFPDupeFilter +DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter' +# 序列化 +SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat" +#自动限速(控制爬取速度，降低对方服务器压力) +AUTOTHROTTLE_ENABLED = True + +mongodb = { + 'host': '120.24.87.40', + 'port': 27017, + 'username': "admin", + 'password': "123456" +} + + +cookies = { + 'shshshfpa': '88442a32-4008-3016-2ed4-5f00ba6e02a6-1583306994', + '__jdu': '1645417385963897370071', + 'shshshfpb': 'eKzbp55ekVbG%2BmXNii7FmnA%3D%3D', + 'unpl': 'JF8EAKpnNSttC0hdDR9XGhMQTw5XW15YGURUPDQCVFwIHFEHGQoSExZ7XlVdXhRKFR9vZxRXXlNKUQ4fBCsSE3tdVV9cD0gVBWduNWRtW0tkBCsCHBcUTl1SXFQMQxABZm8DVltZSlIFKzIcEhl7bWRbXQlKFQVpZAVXbVl7VgQaAB8XFEJcU24cZk4XAmZlSFRaXU9RBR0AEhYYTF9dVlsKTRYCaWc1VlReQ1I1GA', + '__jdv': '76161171|baidu|-|organic|notset|1648984819662', + 'areaId': '20', + 'PCSYCityID': 'CN_450000_450300_0', + 'ipLoc-djd': '20-1726-22884-51455', + 'pinId': 'bxSvmv8CNc-KKJVv2AYoCLV9-x-f3wj7', + 'pin': 'jd_701eaef2a29ff', + 'unick': '%E6%9D%9C%E6%92%B0%E4%B8%AD%E7%9A%84%E6%9C%AA%E5%90%8D', + '_tp': 'Wy5hpT1Zse8ScnmqROf6D38%2BJ8IK3ESqvwXymUySsLE%3D', + '_pst': 'jd_701eaef2a29ff', + 'shshshfp': '9dc45f7c100e4d56fc0ddb7d96bc59ab', + '__jdc': '122270672', + '__jda': '122270672.1645417385963897370071.1645417385.1649430078.1649491583.20', + 'thor': 'FC76D1F441FDD23810222D65B21A1E62936594BACBBD52CF88EB830809C78B7F75DEEDA34458FE020C54AF1DD6E55F389B2063BCCBBD829B977631DD6FB67A17BA374FEFAA00AF1C8264A11F080FA449884B327D73A31031D1F35232730745A6BD1570FDB90E15A4002FCBF4C8CDED1F588BFA29B2272823A7263C9A88F289B84E2075F1A710BED202A651BD7ABBC09D354497FAFBB4A0A7CBDC9590803F6162', + 'ceshi3.com': '000', + 'token': '5a5e7384ead314efe3237efb8d9825fb,3,916384', + '__tk': 'OINnrcq4NDN5NiuEqcrdriKiOcJ5sINEsca4rfKiqIhgNLbhOIN4sG,3,916384', + 'ip_cityCode': '1726', + 'wlfstk_smdl': 'xrr3qwmzs7zoj7y6kbt5nfkai2clqo2n', + 'shshshsID': '3c84311b6180b15c5bf06ba762b90ce2_7_1649491714829', + '__jdb': '122270672.15.1645417385963897370071|20.1649491583', + '3AB9D23F7A4B3C9B': '3WE3SSBAN2EJKLE3VP6ZK2I4UOCXTYVZTD7O46KFL2S7J2UVXVZ7IJGBJKC4S3RAWL2DRAMRLPN63TK3LHWTA5JVQQ', +} + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'jdSkuSpider (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +DOWNLOAD_DELAY = 1 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +DEFAULT_REQUEST_HEADERS = { + 'authority': 'item.jd.com', + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'accept-language': 'zh,zh-CN;q=0.9', + 'cache-control': 'max-age=0', + # Requests sorts cookies= alphabetically + # 'cookie': 'shshshfpa=88442a32-4008-3016-2ed4-5f00ba6e02a6-1583306994; __jdu=1645417385963897370071; shshshfpb=eKzbp55ekVbG%2BmXNii7FmnA%3D%3D; unpl=JF8EAKpnNSttC0hdDR9XGhMQTw5XW15YGURUPDQCVFwIHFEHGQoSExZ7XlVdXhRKFR9vZxRXXlNKUQ4fBCsSE3tdVV9cD0gVBWduNWRtW0tkBCsCHBcUTl1SXFQMQxABZm8DVltZSlIFKzIcEhl7bWRbXQlKFQVpZAVXbVl7VgQaAB8XFEJcU24cZk4XAmZlSFRaXU9RBR0AEhYYTF9dVlsKTRYCaWc1VlReQ1I1GA; __jdv=76161171|baidu|-|organic|notset|1648984819662; areaId=20; PCSYCityID=CN_450000_450300_0; ipLoc-djd=20-1726-22884-51455; pinId=bxSvmv8CNc-KKJVv2AYoCLV9-x-f3wj7; pin=jd_701eaef2a29ff; unick=%E6%9D%9C%E6%92%B0%E4%B8%AD%E7%9A%84%E6%9C%AA%E5%90%8D; _tp=Wy5hpT1Zse8ScnmqROf6D38%2BJ8IK3ESqvwXymUySsLE%3D; _pst=jd_701eaef2a29ff; shshshfp=9dc45f7c100e4d56fc0ddb7d96bc59ab; __jdc=122270672; __jda=122270672.1645417385963897370071.1645417385.1649430078.1649491583.20; thor=FC76D1F441FDD23810222D65B21A1E62936594BACBBD52CF88EB830809C78B7F75DEEDA34458FE020C54AF1DD6E55F389B2063BCCBBD829B977631DD6FB67A17BA374FEFAA00AF1C8264A11F080FA449884B327D73A31031D1F35232730745A6BD1570FDB90E15A4002FCBF4C8CDED1F588BFA29B2272823A7263C9A88F289B84E2075F1A710BED202A651BD7ABBC09D354497FAFBB4A0A7CBDC9590803F6162; ceshi3.com=000; token=5a5e7384ead314efe3237efb8d9825fb,3,916384; __tk=OINnrcq4NDN5NiuEqcrdriKiOcJ5sINEsca4rfKiqIhgNLbhOIN4sG,3,916384; ip_cityCode=1726; wlfstk_smdl=xrr3qwmzs7zoj7y6kbt5nfkai2clqo2n; shshshsID=3c84311b6180b15c5bf06ba762b90ce2_7_1649491714829; __jdb=122270672.15.1645417385963897370071|20.1649491583; 3AB9D23F7A4B3C9B=3WE3SSBAN2EJKLE3VP6ZK2I4UOCXTYVZTD7O46KFL2S7J2UVXVZ7IJGBJKC4S3RAWL2DRAMRLPN63TK3LHWTA5JVQQ', + 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36', +} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'jdSkuSpider.middlewares.JdskuspiderSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +DOWNLOADER_MIDDLEWARES = { + 'jdSkuSpider.middlewares.ProxyMiddleWare':542, + 'jdSkuSpider.middlewares.JdskuspiderDownloaderMiddleware': 543, +} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'scrapy_redis.pipelines.RedisPipeline':400, + 'jdSkuSpider.pipelines.JdskuspiderPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/jdSkuSpider/jdSkuSpider/spiders/__init__.py b/jdSkuSpider/jdSkuSpider/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/jdSkuSpider/jdSkuSpider/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/jdSkuSpider/jdSkuSpider/spiders/jdsku.py b/jdSkuSpider/jdSkuSpider/spiders/jdsku.py new file mode 100644 index 0000000..753ee84 --- /dev/null +++ b/jdSkuSpider/jdSkuSpider/spiders/jdsku.py @@ -0,0 +1,119 @@ +import scrapy +import re + +import redis +from scrapy_redis.spiders import RedisSpider +from pymongo import MongoClient + +from ..items import * +from ..settings import REDIS_HOST,REDIS_PORT,REDIS_DB,REDIS_PARAMS, mongodb + +class JdskuSpider(RedisSpider): + name = 'jdsku' + allowed_domains = ['jd.com'] + # start_urls = ['https://item.jd.com/100018781645.html#none'] + redis_key = 'jd:start_urls' + + redis_connection = redis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PARAMS.get('password'), + db=REDIS_DB, + charset='UTF-8', + encoding='UTF-8') + + mongo_connection = MongoClient( + host=mongodb.get('host'), + port=mongodb.get('port'), + username=mongodb.get('username'), + password=mongodb.get('password')) + + # 解决AttributeError: 'JdskuSpider' object has no attribute 'make_requests_from_url' + def make_requests_from_url(self,url): + #dont_filter=True带表不去重, 默认False, 去重 + return scrapy.Request(url, + meta={'url':url}, #将url传给parse + dont_filter=self.settings.get("DONT_FILTER"), + cookies=self.settings.get('cookies')) + + def parse(self, response): + + sku = response.meta['url'] + sku_pattern = re.compile("[0-9]+") + sku = re.search(sku_pattern, sku).group() + + bitWidth = -1 + model = -1 + memoryType = -1 + memoryCapacity = -1 + title = response.xpath('//*[@id="parameter-brand"]/li/a/text()').extract_first() + + print("=========== title =================") + print(response.xpath('/html')) + print("=========== title =================") + li_list = response.xpath('//*[@id="detail"]/div[2]/div[1]/div[1]/ul[2]/li') + print("***********************************************") + for li in li_list: + value = li.xpath('@title').re(".*")[0] + key = li.re(">.*<")[0][1:-1].replace("：", ":").replace(": {}".format(value), "").replace(":{}".format(value), "") + print({key:value}) + if key == '显存位宽': + bitWidth = value + elif key =='显卡型号': + model = value + elif key =='显存类型': + memoryType = value + elif key == '显存容量': + memoryCapacity = value + elif key == '商品名称': + title = value + print("***********************************************") + + item = JdskuspiderItem( + sku=sku, + title=title, + model=model, + memoryType=memoryType, + memoryCapacity=memoryCapacity, + bitWidth=bitWidth + ) + print("=========== 现在的sku ==========") + print(sku) + + if bitWidth==-1 and memoryType==-1 and memoryCapacity==-1: + print("========== no insert into mongodb ==============") + if "window.location.href='https://passport.jd.com" in response.xpath('/html').xpath('string(.)').extract_first(): + print("window.location.href页面") + print("sku重发redis") + self.redis_connection.lpush("jd:start_urls", 'https://item.jd.com/{}.html'.format(sku)) + else: + print("非显卡商品, insert -1") + mongoDB = self.mongo_connection['admin'] + mongoCollection = mongoDB['jd'] + mongoCollection.find_one_and_update( + filter={'sku': sku}, + update={'$set': { + 'title': title, + 'model': model, + 'memoryType': memoryType, + 'memoryCapacity': memoryCapacity, + 'bitWidth': bitWidth + }} + ) + else: + print("========== insert into mongodb ==============") + mongoDB = self.mongo_connection['admin'] + mongoCollection = mongoDB['jd'] + mongoCollection.find_one_and_update( + filter={'sku':sku}, + update={'$set':{ + 'title':title, + 'model':model, + 'memoryType':memoryType, + 'memoryCapacity':memoryCapacity, + 'bitWidth': bitWidth + }} + ) + + yield item + diff --git a/jdSkuSpider/scrapy.cfg b/jdSkuSpider/scrapy.cfg new file mode 100644 index 0000000..6111bf9 --- /dev/null +++ b/jdSkuSpider/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = jdSkuSpider.settings + +[deploy] +#url = http://localhost:6800/ +project = jdSkuSpider