proxyherd 1.0.0__tar.gz

proxyherd-1.0.0/LICENCE

@@ -0,0 +1,17 @@
+# The MIT License (MIT)
+
+Copyright (c) 2016-present Fabien Vauchelles
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.

proxyherd-1.0.0/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include MANIFEST.in
+recursive-include proxyherd *
+recursive-include scrapoxy *

proxyherd-1.0.0/PKG-INFO

@@ -0,0 +1,40 @@
+Metadata-Version: 2.4
+Name: proxyherd
+Version: 1.0.0
+Summary: Python API and Scrapy middlewares for ProxyHerd, the open-source super proxy manager
+Home-page: https://github.com/proxyherd/proxyherd
+Author: ProxyHerd maintainers
+License: MIT
+Keywords: crawler,crawling,proxyherd,scrapoxy,scrapy,scraper,scraping,webscraping,proxy
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Framework :: Scrapy
+Description-Content-Type: text/markdown
+License-File: LICENCE
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: summary
+
+# Python API for ProxyHerd
+
+Scrapy middlewares and a small REST client for [ProxyHerd](https://proxyherd.github.io/proxyherd/),
+the open-source super proxy manager.
+
+```shell
+pip install proxyherd
+```
+
+Documentation: https://proxyherd.github.io/proxyherd/integration/python/scrapy/guide
+
+## Migrating from the `scrapoxy` package
+
+`proxyherd` is the continuation of the `scrapoxy` Python package (community fork of Scrapoxy 4.17.4).
+The module `proxyherd` exposes the same middlewares; the settings are now prefixed `PROXYHERD_`
+(`PROXYHERD_MASTER`, `PROXYHERD_API`, `PROXYHERD_USERNAME`, `PROXYHERD_PASSWORD`, ...).
+The old `SCRAPOXY_` settings and the `scrapoxy` module keep working as deprecated aliases.

proxyherd-1.0.0/README.md

@@ -0,0 +1,17 @@
+# Python API for ProxyHerd
+
+Scrapy middlewares and a small REST client for [ProxyHerd](https://proxyherd.github.io/proxyherd/),
+the open-source super proxy manager.
+
+```shell
+pip install proxyherd
+```
+
+Documentation: https://proxyherd.github.io/proxyherd/integration/python/scrapy/guide
+
+## Migrating from the `scrapoxy` package
+
+`proxyherd` is the continuation of the `scrapoxy` Python package (community fork of Scrapoxy 4.17.4).
+The module `proxyherd` exposes the same middlewares; the settings are now prefixed `PROXYHERD_`
+(`PROXYHERD_MASTER`, `PROXYHERD_API`, `PROXYHERD_USERNAME`, `PROXYHERD_PASSWORD`, ...).
+The old `SCRAPOXY_` settings and the `scrapoxy` module keep working as deprecated aliases.

proxyherd-1.0.0/proxyherd/__init__.py

@@ -0,0 +1,5 @@
+from proxyherd.api import ProxyHerdApi
+from proxyherd.blacklist import BlacklistDownloaderMiddleware
+from proxyherd.proxy import ProxyDownloaderMiddleware
+from proxyherd.scale import ScaleSpiderMiddleware
+from proxyherd.sticky import StickySpiderMiddleware

proxyherd-1.0.0/proxyherd/api.py

@@ -0,0 +1,72 @@
+from base64 import b64encode
+from urllib.request import Request, urlopen
+from typing import TypedDict, List
+
+import json
+
+
+def is_proxy_online(proxy):
+    return (
+        proxy['status'] == 'STARTED' and
+        proxy['fingerprint'] and
+        not proxy['removing']
+    )
+
+
+class IProxyToRemove(TypedDict):
+    id: str
+    force: bool
+
+
+class ProxyHerdApi:
+    def __init__(self, url: str, username: str, password: str):
+        self._url = f"{url}/scraper"
+        self._token = b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8")
+
+    def get_project(self):
+        req = Request(
+            url=f"{self._url}/project",
+            headers={
+                "Authorization": f"Basic {self._token}"
+            }
+        )
+        contents = urlopen(req).read()
+        return json.loads(contents)
+
+    def set_project_status(self, status: str):
+        payload = {"status": status}
+        req = Request(
+            url = f"{self._url}/project/status",
+            method="POST",
+            headers={
+                "Authorization": f"Basic {self._token}",
+                "Content-Type": "application/json"
+            },
+            data=json.dumps(payload).encode("utf-8")
+        )
+        urlopen(req).read()
+
+    def get_all_project_connectors_and_proxies(self):
+        req = Request(
+            url=f"{self._url}/project/connectors",
+            headers={
+                "Authorization": f"Basic {self._token}"
+            }
+        )
+        contents = urlopen(req).read()
+        return json.loads(contents)
+
+    def ask_proxies_to_remove(self, proxies_ids: List[IProxyToRemove]):
+        req = Request(
+            url=f"{self._url}/project/proxies/remove",
+            method="POST",
+            headers={
+                "Authorization": f"Basic {self._token}",
+                "Content-Type": "application/json"
+            },
+            data=json.dumps(proxies_ids).encode("utf-8")
+        )
+        urlopen(req).read()
+
+# Backward-compatible alias (the package was named scrapoxy before 1.0.0)
+ScrapoxyApi = ProxyHerdApi

proxyherd-1.0.0/proxyherd/blacklist.py

@@ -0,0 +1,76 @@
+from random import randrange
+from scrapy.exceptions import IgnoreRequest
+from proxyherd.api import ProxyHerdApi, is_proxy_online
+from time import sleep
+
+
+class BlacklistError(Exception):
+    def __init__(self, response, message, *args, **kwargs):
+        super(BlacklistError, self).__init__(*args, **kwargs)
+
+        self.response = response
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class BlacklistDownloaderMiddleware(object):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler):
+        """Access the settings of the crawler to connect to ProxyHerd.
+        """
+        api = crawler.settings.get("PROXYHERD_API", crawler.settings.get("SCRAPOXY_API"))
+        assert api, "PROXYHERD_API is required"
+
+        username = crawler.settings.get("PROXYHERD_USERNAME", crawler.settings.get("SCRAPOXY_USERNAME"))
+        assert username, "PROXYHERD_USERNAME is required"
+
+        password = crawler.settings.get("PROXYHERD_PASSWORD", crawler.settings.get("SCRAPOXY_PASSWORD"))
+        assert password, "PROXYHERD_PASSWORD is required"
+
+        self._api = ProxyHerdApi(api, username, password)
+
+        self._http_status_codes = crawler.settings.get("PROXYHERD_BLACKLIST_HTTP_STATUS_CODES", crawler.settings.get("SCRAPOXY_BLACKLIST_HTTP_STATUS_CODES", [429, 503]))
+        self._force = crawler.settings.get("PROXYHERD_BLACKLIST_FORCE", crawler.settings.get("SCRAPOXY_BLACKLIST_FORCE", True))
+        self._sleep_min = crawler.settings.get("PROXYHERD_SLEEP_MIN", crawler.settings.get("SCRAPOXY_SLEEP_MIN", 60))
+        self._sleep_max = crawler.settings.get("PROXYHERD_SLEEP_MAX", crawler.settings.get("SCRAPOXY_SLEEP_MAX", 180))
+
+    def process_response(self, request, response, spider):
+        if response.status not in self._http_status_codes:
+            return response
+
+        spider.logger.info("Ignoring Blacklisted response %s: HTTP status %d" % (response.url, response.status))
+
+        id_raw = response.headers.get("x-scrapoxy-proxyname")
+        if not id_raw:
+            raise BlacklistError(response, "No header 'X-Scrapoxy-Proxyname' name in response headers. MITM must be enabled")
+
+        id = id_raw.decode("utf-8")
+
+        alive_count = 0
+        views = self._api.get_all_project_connectors_and_proxies()
+        for view in views:
+            for proxy in view["proxies"]:
+                if (proxy["id"] != id and is_proxy_online(proxy)):
+                    alive_count += 1
+
+        spider.logger.error("Remove instance %s (remaining %d instances)." % (response.url, response.status))
+
+        self._api.ask_proxies_to_remove([
+            {
+                "id": id,
+                "force": self._force
+            }
+        ])
+
+        if alive_count <= 0:
+            delay = randrange(self._sleep_min, self._sleep_max)
+            spider.logger.warn("No instance remaining. Sleep for %d seconds..." % delay)
+            sleep(delay)
+
+        raise IgnoreRequest()

proxyherd-1.0.0/proxyherd/proxy.py

@@ -0,0 +1,40 @@
+from base64 import b64encode
+
+
+class ProxyDownloaderMiddleware(object):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler):
+        master = crawler.settings.get("PROXYHERD_MASTER", crawler.settings.get("SCRAPOXY_MASTER"))
+        if master:
+            self._proxy = f"{master}/?noconnect"
+
+        username = crawler.settings.get("PROXYHERD_USERNAME", crawler.settings.get("SCRAPOXY_USERNAME"))
+        password = crawler.settings.get("PROXYHERD_PASSWORD", crawler.settings.get("SCRAPOXY_PASSWORD"))
+        if username and password:
+            auth_b64 = b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8")
+            self._proxy_auth = (username, password)
+            self._proxy_auth_b64 = f"Basic {auth_b64}"
+
+    def process_request(self, request, spider):
+        if request.meta.get("no-proxy"):
+            return
+
+        # Don"t overwrite with a random one (server-side state for IP)
+        if hasattr(self, "_proxy"):
+            request.meta["proxy"] = self._proxy
+
+            if hasattr(self, "_proxy_auth"):
+                request.headers["proxy-authorization"] = self._proxy_auth_b64
+
+            if request.meta.get("impersonate"):
+                impersonate_args = request.meta.get("impersonate_args", {})
+                impersonate_args["verify"] = False
+
+                if hasattr(self, "_proxy_auth"):
+                    impersonate_args["proxy_auth"] = self._proxy_auth
+
+                request.meta["impersonate_args"] = impersonate_args

proxyherd-1.0.0/proxyherd/scale.py

@@ -0,0 +1,42 @@
+from scrapy import signals
+from proxyherd.api import ProxyHerdApi
+from time import sleep
+
+
+class ScaleSpiderMiddleware(object):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler):
+        api = crawler.settings.get("PROXYHERD_API", crawler.settings.get("SCRAPOXY_API"))
+        assert api, "PROXYHERD_API is required"
+
+        username = crawler.settings.get("PROXYHERD_USERNAME", crawler.settings.get("SCRAPOXY_USERNAME"))
+        assert username, "PROXYHERD_USERNAME is required"
+
+        password = crawler.settings.get("PROXYHERD_PASSWORD", crawler.settings.get("SCRAPOXY_PASSWORD"))
+        assert password, "PROXYHERD_PASSWORD is required"
+
+        self._api = ProxyHerdApi(api, username, password)
+
+        self._delay = crawler.settings.get("PROXYHERD_WAIT_FOR_SCALE", crawler.settings.get("SCRAPOXY_WAIT_FOR_SCALE", 120))
+
+        crawler.signals.connect(self.spider_opened, signals.spider_opened)
+        crawler.signals.connect(self.spider_closed, signals.spider_closed)
+
+
+    def spider_opened(self, spider):
+        spider.logger.info("[ScaleSpiderMiddleware] Upscale ProxyHerd")
+
+        self._api.set_project_status("HOT")
+
+        spider.logger.info("[ScaleSpiderMiddleware] Sleeping %d seconds to finish upscale" % self._delay)
+        sleep(self._delay)
+
+
+    def spider_closed(self, spider):
+        spider.logger.info("[ScaleSpiderMiddleware] Downscale ProxyHerd")
+
+        self._api.set_project_status("OFF")

proxyherd-1.0.0/proxyherd/sticky.py

@@ -0,0 +1,14 @@
+from scrapy import Request
+
+
+class StickySpiderMiddleware:
+    # Propagate proxyname between responses and requests to have sticky session
+    def process_spider_output(self, response, result, spider):
+        proxyname = response.headers.get("x-scrapoxy-proxyname")
+        if proxyname:
+            for r in result:
+                if isinstance(r, Request):
+                    r.headers["x-scrapoxy-proxyname"] = proxyname
+                yield r
+        else:
+            yield from result

proxyherd-1.0.0/proxyherd.egg-info/PKG-INFO

@@ -0,0 +1,40 @@
+Metadata-Version: 2.4
+Name: proxyherd
+Version: 1.0.0
+Summary: Python API and Scrapy middlewares for ProxyHerd, the open-source super proxy manager
+Home-page: https://github.com/proxyherd/proxyherd
+Author: ProxyHerd maintainers
+License: MIT
+Keywords: crawler,crawling,proxyherd,scrapoxy,scrapy,scraper,scraping,webscraping,proxy
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Framework :: Scrapy
+Description-Content-Type: text/markdown
+License-File: LICENCE
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: summary
+
+# Python API for ProxyHerd
+
+Scrapy middlewares and a small REST client for [ProxyHerd](https://proxyherd.github.io/proxyherd/),
+the open-source super proxy manager.
+
+```shell
+pip install proxyherd
+```
+
+Documentation: https://proxyherd.github.io/proxyherd/integration/python/scrapy/guide
+
+## Migrating from the `scrapoxy` package
+
+`proxyherd` is the continuation of the `scrapoxy` Python package (community fork of Scrapoxy 4.17.4).
+The module `proxyherd` exposes the same middlewares; the settings are now prefixed `PROXYHERD_`
+(`PROXYHERD_MASTER`, `PROXYHERD_API`, `PROXYHERD_USERNAME`, `PROXYHERD_PASSWORD`, ...).
+The old `SCRAPOXY_` settings and the `scrapoxy` module keep working as deprecated aliases.

proxyherd-1.0.0/proxyherd.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+LICENCE
+MANIFEST.in
+README.md
+setup.cfg
+setup.py
+proxyherd/__init__.py
+proxyherd/api.py
+proxyherd/blacklist.py
+proxyherd/proxy.py
+proxyherd/scale.py
+proxyherd/sticky.py
+proxyherd.egg-info/PKG-INFO
+proxyherd.egg-info/SOURCES.txt
+proxyherd.egg-info/dependency_links.txt
+proxyherd.egg-info/top_level.txt
+scrapoxy/__init__.py
+scrapoxy/api.py
+scrapoxy/blacklist.py
+scrapoxy/proxy.py
+scrapoxy/scale.py
+scrapoxy/sticky.py

proxyherd-1.0.0/proxyherd.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

proxyherd-1.0.0/proxyherd.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+proxyherd
+scrapoxy

proxyherd-1.0.0/scrapoxy/__init__.py

@@ -0,0 +1,18 @@
+"""Compatibility shim.
+
+The ``scrapoxy`` Python package continues as ``proxyherd`` (community fork of Scrapoxy 4.17.4).
+Importing ``scrapoxy`` still works but is deprecated; import ``proxyherd`` instead.
+"""
+import warnings
+
+from proxyherd.api import ProxyHerdApi, ProxyHerdApi as ScrapoxyApi, is_proxy_online  # noqa: F401
+from proxyherd.blacklist import BlacklistDownloaderMiddleware  # noqa: F401
+from proxyherd.proxy import ProxyDownloaderMiddleware  # noqa: F401
+from proxyherd.scale import ScaleSpiderMiddleware  # noqa: F401
+from proxyherd.sticky import StickySpiderMiddleware  # noqa: F401
+
+warnings.warn(
+    "The 'scrapoxy' module is deprecated; import 'proxyherd' instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)

proxyherd-1.0.0/scrapoxy/api.py

@@ -0,0 +1,2 @@
+from proxyherd.api import *  # noqa: F401,F403
+from proxyherd.api import ProxyHerdApi as ScrapoxyApi  # noqa: F401

proxyherd-1.0.0/scrapoxy/blacklist.py

@@ -0,0 +1 @@
+from proxyherd.blacklist import *  # noqa: F401,F403

proxyherd-1.0.0/scrapoxy/proxy.py

@@ -0,0 +1 @@
+from proxyherd.proxy import *  # noqa: F401,F403

proxyherd-1.0.0/scrapoxy/scale.py

@@ -0,0 +1 @@
+from proxyherd.scale import *  # noqa: F401,F403

proxyherd-1.0.0/scrapoxy/sticky.py

@@ -0,0 +1 @@
+from proxyherd.sticky import *  # noqa: F401,F403

proxyherd-1.0.0/setup.cfg

@@ -0,0 +1,7 @@
+[metadata]
+description-file = README.md
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

proxyherd-1.0.0/setup.py

@@ -0,0 +1,24 @@
+try:
+    from setuptools import setup, find_packages
+except ImportError:
+    from distutils.core import setup, find_packages
+
+
+setup(
+    name='proxyherd',
+    packages=find_packages(),
+    install_requires=[],
+    version='1.0.0',
+    description='Python API and Scrapy middlewares for ProxyHerd, the open-source super proxy manager',
+    long_description=open('README.md').read(),
+    long_description_content_type='text/markdown',
+    author='ProxyHerd maintainers',
+    url='https://github.com/proxyherd/proxyherd',
+    license='MIT',
+    keywords=['crawler', 'crawling', 'proxyherd', 'scrapoxy', 'scrapy', 'scraper', 'scraping', 'webscraping', 'proxy'],
+    classifiers=[
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3',
+        'Framework :: Scrapy',
+    ],
+)