grabpy 0.0.1__py3-none-any.whl

grabpy/__init__.py

@@ -0,0 +1,3 @@
+from .scraper import Grabber
+
+__all__ = ['Grabber']

grabpy/request.py

@@ -0,0 +1,27 @@
+import requests
+import sys
+
+
+class Requester:
+    def __init__(self, retries: int):
+        """Set retries to -1 to retry indefinitely"""
+        self.retries = retries
+
+    def fetch(self, url: str) -> bytes:
+        retries: int = self.retries
+
+        while retries:
+            try:
+                response = requests.get(url)
+                response.raise_for_status()
+            except requests.exceptions.RequestException as err:
+                print(err, file=sys.stderr)
+
+                if err.response.status_code == 404:
+                    break
+
+                retries -= 1
+            else:
+                return response.content
+
+        return b''

grabpy/robots.py

@@ -0,0 +1,60 @@
+from urllib.robotparser import RobotFileParser
+import urllib.parse
+import time
+from functools import lru_cache
+import sys
+
+
+class RobotsParser:
+    def __init__(self, useragent: str) -> None:
+        self.useragent: str = useragent
+
+    def __str__(self) -> str:
+        return f'RobotsParser-{self.useragent}'
+
+    @staticmethod
+    def _extract_url_base(url: str) -> str:
+        result = urllib.parse.urlparse(url)
+        netloc = result.netloc
+        scheme = result.scheme
+
+        if not all([netloc, scheme]):
+            raise ValueError(f'Invalid url schema: {url}')
+
+        return f'{scheme}://{netloc}'
+
+    @lru_cache(maxsize=128, typed=True)
+    def _get_parser(self, url: str) -> RobotFileParser:
+        base = self._extract_url_base(url)
+        url = urllib.parse.urljoin(base, 'robots.txt')
+
+        rp = RobotFileParser()
+        rp.set_url(url)
+        rp.read()
+
+        return rp
+
+    def _can_scrape(self, parser: RobotFileParser, url: str) -> bool:
+        return parser.can_fetch(self.useragent, url)
+
+    def _scrape_delay(self, parser: RobotFileParser) -> None:
+        delay = parser.crawl_delay(self.useragent)
+
+        if not delay:
+            return
+
+        time.sleep(float(delay))
+
+    def be_respectful(self, url: str) -> bool:
+        try:
+            parser = self._get_parser(url)
+        except ValueError as err:
+            print(err, file=sys.stderr)
+            return False
+
+        if not self._can_scrape(parser, url):
+            return False
+
+        self._scrape_delay(parser)
+
+        return True

grabpy/scraper.py

@@ -0,0 +1,22 @@
+from .request import Requester
+from .robots import RobotsParser
+from functools import lru_cache
+
+
+class Grabber:
+    def __init__(self, useragent: str, retries: int = 3) -> None:
+        self.robots_parser = RobotsParser(useragent)
+        self.requester = Requester(retries)
+
+    def __enter__(self) -> 'Grabber':
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        return
+
+    @lru_cache(maxsize=8, typed=True)
+    def get(self, url: str) -> bytes:
+        if not self.robots_parser.be_respectful(url):
+            return b''
+
+        return self.requester.fetch(url)

grabpy-0.0.1.dist-info/METADATA

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: grabpy
+Version: 0.0.1
+Summary: Simple package for requesting web pages that respects robots.txt
+Author-email: Riain Ó Tuathail <rotuathail01@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/belfastkeyboard/grabpy
+Project-URL: Issues, https://github.com/belfastkeyboard/grabpy/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENCE
+Requires-Dist: requests==2.32.5
+Dynamic: license-file
+
+# Grabpy
+
+Simple respectful web scraper

grabpy-0.0.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+grabpy/__init__.py,sha256=lh27thZnh_FCMi4YDcgyDWtD8zUP9yc3IdD7bR3qrx4,52
+grabpy/request.py,sha256=NyfgXf1JSTKswNqknfZVhVP4iklU3U0FFd2Iy8iPBlk,671
+grabpy/robots.py,sha256=GE08Qi0wZxakpUs0dIlt4Ym_8hcNdaWIW9OworCWcDE,1551
+grabpy/scraper.py,sha256=MTSGG1pTTbSQeQaiclXIguN0eqD7WpQi9h0hqdVtz2I,606
+grabpy-0.0.1.dist-info/licenses/LICENCE,sha256=0ldxwBhIbfESmnAHjCsiGAutVM7VPltW6oC_odQeK_g,1060
+grabpy-0.0.1.dist-info/METADATA,sha256=zsaIpDHccnZGUxU1EAH42Uc3hYUGOkA086xuH5UruiE,615
+grabpy-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+grabpy-0.0.1.dist-info/top_level.txt,sha256=Bdj1abPGSUhEP9o64HuSrA1GKMLbAgTEWke7xEv6XYQ,7
+grabpy-0.0.1.dist-info/RECORD,,

grabpy-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

grabpy-0.0.1.dist-info/licenses/LICENCE

@@ -0,0 +1,19 @@
+Copyright (c) 2025 Riain Ó Tuathail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

grabpy-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+grabpy