msaas-scraper-framework 0.1.0__py3-none-any.whl

msaas_scraper_framework-0.1.0.dist-info/METADATA

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: msaas-scraper-framework
+Version: 0.1.0
+Summary: Generic web scraper framework with registry, rate limiting, and composable filters
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: msaas-api-core
+Requires-Dist: msaas-errors
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'

msaas_scraper_framework-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+scraper_framework/__init__.py,sha256=fss4om1vREfIvsDBcmdBqKWdiGbikInQ_xTEho68tEE,1305
+scraper_framework/base.py,sha256=fBZjgd8px7blay4-NgQyyrYvTSqpBCZ3IBeyLdU8gJ8,5672
+scraper_framework/filters.py,sha256=pynhfV0Tvc_2-hSNTEjSBg0gxqvh7HfwejJVDtEjbmA,3680
+scraper_framework/rate_limiter.py,sha256=rS2Uc5tWVx6YZc1Y1FJfyF4MsYSRNwzAubkv_RFMSnY,3335
+scraper_framework/registry.py,sha256=aSws_jA2q-uIH15C6yXyJlfge5k3CTjMUnEdNL9TzO0,3638
+scraper_framework/result.py,sha256=arDqiYaHz83C3CU88yX44x9z342p-jNQScNOhmxQlrU,1935
+msaas_scraper_framework-0.1.0.dist-info/METADATA,sha256=rLXQLT-0ELmAmf9MkRsKlt__RfsT8eP-3w2Lrn4L_9g,471
+msaas_scraper_framework-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+msaas_scraper_framework-0.1.0.dist-info/RECORD,,

msaas_scraper_framework-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

scraper_framework/__init__.py

@@ -0,0 +1,52 @@
+"""scraper-framework: Generic web scraper framework with registry, rate limiting, and filters.
+
+Usage::
+
+    from scraper_framework import BaseScraper, ScraperResult, ScraperRegistry
+
+    registry = ScraperRegistry()
+
+    @registry.register("example")
+    class ExampleScraper(BaseScraper):
+        source = "example"
+        id_prefix = "ex"
+
+        def _run(self) -> ScraperResult:
+            with self.make_client() as client:
+                resp = client.get("https://example.com/api")
+                self.result.total_fetched += len(resp.json())
+                self.sleep()
+            return self.result
+
+    scraper = registry.get("example")
+    result = scraper.run()
+    print(result.to_dict())
+"""
+
+from scraper_framework.base import BaseScraper
+from scraper_framework.filters import (
+    all_match,
+    any_match,
+    keyword_filter,
+    length_filter,
+    negate,
+    none_match,
+    regex_filter,
+)
+from scraper_framework.rate_limiter import RateLimiter
+from scraper_framework.registry import ScraperRegistry
+from scraper_framework.result import ScraperResult
+
+__all__ = [
+    "BaseScraper",
+    "RateLimiter",
+    "ScraperRegistry",
+    "ScraperResult",
+    "all_match",
+    "any_match",
+    "keyword_filter",
+    "length_filter",
+    "negate",
+    "none_match",
+    "regex_filter",
+]

scraper_framework/base.py

@@ -0,0 +1,170 @@
+"""Generic base scraper class.
+
+Provides the abstract skeleton that concrete scrapers implement. Handles
+lifecycle (setup, run, teardown), HTTP client creation, rate limiting, and
+result tracking.
+
+Example subclass::
+
+    from scraper_framework import BaseScraper, ScraperResult
+
+    class MyScraper(BaseScraper):
+        source = "my-source"
+        id_prefix = "ms"
+
+        def _run(self) -> ScraperResult:
+            with self.make_client() as client:
+                resp = client.get("https://example.com/api/items")
+                items = resp.json()
+                self.result.total_fetched += len(items)
+                # ... process items ...
+                self.sleep()
+            return self.result
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+
+import httpx
+
+from scraper_framework.rate_limiter import RateLimiter
+from scraper_framework.result import ScraperResult
+
+
+class BaseScraper(ABC):
+    """Abstract base class for all scrapers.
+
+    Subclasses **must** set ``source`` and implement :meth:`_run`.
+
+    Class attributes:
+        source: Identifier for this scraper (e.g. ``"greenhouse"``).
+        id_prefix: Short prefix for generated record IDs (e.g. ``"gh"``).
+        rate_limit_range: Default ``(min, max)`` seconds between requests.
+    """
+
+    source: str = ""
+    id_prefix: str = ""
+    rate_limit_range: tuple[float, float] = (2.0, 5.0)
+
+    def __init__(self, *, config: dict[str, Any] | None = None) -> None:
+        self.config: dict[str, Any] = config or {}
+        self.result = ScraperResult(source=self.source)
+        self.rate_limiter = RateLimiter(
+            min_seconds=self.rate_limit_range[0],
+            max_seconds=self.rate_limit_range[1],
+        )
+        try:
+            import structlog
+
+            self.log = structlog.get_logger(f"scraper.{self.source}").bind(source=self.source)
+        except ImportError:
+            self.log = logging.getLogger(f"scraper.{self.source}")
+
+    # ------------------------------------------------------------------
+    # Lifecycle hooks (override in subclasses if needed)
+    # ------------------------------------------------------------------
+
+    def setup(self) -> None:
+        """Called before :meth:`_run`. Override for one-time initialisation."""
+
+    def teardown(self) -> None:
+        """Called after :meth:`_run` completes (even on error). Override for cleanup."""
+
+    # ------------------------------------------------------------------
+    # Abstract
+    # ------------------------------------------------------------------
+
+    @abstractmethod
+    def _run(self) -> ScraperResult:
+        """Execute the scraping logic.
+
+        Must populate ``self.result`` and return it. Implementations should
+        call ``self.sleep()`` between HTTP requests to respect rate limits.
+        """
+        ...
+
+    # ------------------------------------------------------------------
+    # Public entry point
+    # ------------------------------------------------------------------
+
+    def run(
+        self,
+        *,
+        on_before: Callable[[BaseScraper], None] | None = None,
+        on_after: Callable[[BaseScraper, ScraperResult], None] | None = None,
+    ) -> ScraperResult:
+        """Public entry point: setup, run scraper, teardown, finalize result.
+
+        Args:
+            on_before: Optional callback invoked after :meth:`setup` but
+                before :meth:`_run`.
+            on_after: Optional callback invoked after :meth:`_run` with the
+                result.
+
+        Returns:
+            The populated :class:`ScraperResult`.
+        """
+        self.setup()
+        if on_before:
+            on_before(self)
+        try:
+            self._run()
+        except Exception:
+            self.result.errors += 1
+            self.log.exception("Scraper %s failed", self.source)
+            raise
+        finally:
+            self.teardown()
+            self.result.finish()
+        if on_after:
+            on_after(self, self.result)
+        return self.result
+
+    # ------------------------------------------------------------------
+    # Utilities
+    # ------------------------------------------------------------------
+
+    def sleep(self, range_override: tuple[float, float] | None = None) -> float:
+        """Rate-limiting sleep between requests.
+
+        Args:
+            range_override: Optional ``(min, max)`` seconds to override the
+                default rate limit range.
+
+        Returns:
+            Actual seconds slept.
+        """
+        return self.rate_limiter.wait(override=range_override)
+
+    def make_client(self, **kwargs: Any) -> httpx.Client:
+        """Create an :class:`httpx.Client` with sensible defaults.
+
+        Defaults:
+            - ``timeout``: 30 seconds
+            - ``follow_redirects``: True
+
+        All keyword arguments are forwarded to ``httpx.Client``, overriding
+        the defaults.
+        """
+        defaults: dict[str, Any] = {"timeout": 30.0, "follow_redirects": True}
+        defaults.update(kwargs)
+        return httpx.Client(**defaults)
+
+    def make_async_client(self, **kwargs: Any) -> httpx.AsyncClient:
+        """Create an :class:`httpx.AsyncClient` with sensible defaults.
+
+        Same defaults as :meth:`make_client`.
+        """
+        defaults: dict[str, Any] = {"timeout": 30.0, "follow_redirects": True}
+        defaults.update(kwargs)
+        return httpx.AsyncClient(**defaults)
+
+    @staticmethod
+    def now_iso() -> str:
+        """Current UTC timestamp in ISO-8601 format."""
+        return datetime.now(UTC).isoformat()

scraper_framework/filters.py

@@ -0,0 +1,135 @@
+"""Composable filter functions for scraper pipelines.
+
+Filters are plain functions with the signature ``(text: str) -> bool`` so they
+can be combined freely with :func:`all_match`, :func:`any_match`, and
+:func:`none_match`.
+
+Example::
+
+    from scraper_framework.filters import (
+        regex_filter, keyword_filter, all_match, none_match,
+    )
+
+    is_senior = keyword_filter({"senior", "staff", "lead"})
+    not_excluded = none_match(keyword_filter({"intern", "junior"}))
+    has_python = regex_filter(r"\\bpython\\b")
+
+    title = "Senior Python Engineer"
+    if is_senior(title) and not_excluded(title) and has_python(title):
+        print("Relevant!")
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+
+type TextFilter = Callable[[str], bool]
+
+
+# ---------------------------------------------------------------------------
+# Primitive filter builders
+# ---------------------------------------------------------------------------
+
+
+def keyword_filter(
+    keywords: set[str],
+    *,
+    case_sensitive: bool = False,
+) -> TextFilter:
+    """Return a filter that matches if *any* keyword appears in the text.
+
+    Keywords are matched as substrings. For word-boundary matching use
+    :func:`regex_filter` instead.
+
+    Args:
+        keywords: Set of keywords to look for.
+        case_sensitive: Whether matching is case-sensitive.
+    """
+    if not case_sensitive:
+        normalised = {k.lower() for k in keywords}
+
+        def _match(text: str) -> bool:
+            lowered = text.lower()
+            return any(kw in lowered for kw in normalised)
+    else:
+
+        def _match(text: str) -> bool:
+            return any(kw in text for kw in keywords)
+
+    return _match
+
+
+def regex_filter(pattern: str, *, flags: int = re.IGNORECASE) -> TextFilter:
+    """Return a filter that matches if the compiled regex finds a match.
+
+    Args:
+        pattern: A regular expression string.
+        flags: Regex flags (default: ``re.IGNORECASE``).
+    """
+    compiled = re.compile(pattern, flags)
+
+    def _match(text: str) -> bool:
+        return compiled.search(text) is not None
+
+    return _match
+
+
+def length_filter(*, min_length: int = 0, max_length: int | None = None) -> TextFilter:
+    """Return a filter that checks text length.
+
+    Args:
+        min_length: Minimum number of characters.
+        max_length: Maximum number of characters (``None`` for unlimited).
+    """
+
+    def _match(text: str) -> bool:
+        n = len(text)
+        if n < min_length:
+            return False
+        if max_length is not None and n > max_length:
+            return False
+        return True
+
+    return _match
+
+
+# ---------------------------------------------------------------------------
+# Combinators
+# ---------------------------------------------------------------------------
+
+
+def all_match(*filters: TextFilter) -> TextFilter:
+    """Return a filter that passes only if **all** sub-filters pass."""
+
+    def _match(text: str) -> bool:
+        return all(f(text) for f in filters)
+
+    return _match
+
+
+def any_match(*filters: TextFilter) -> TextFilter:
+    """Return a filter that passes if **any** sub-filter passes."""
+
+    def _match(text: str) -> bool:
+        return any(f(text) for f in filters)
+
+    return _match
+
+
+def none_match(*filters: TextFilter) -> TextFilter:
+    """Return a filter that passes only if **no** sub-filter passes."""
+
+    def _match(text: str) -> bool:
+        return not any(f(text) for f in filters)
+
+    return _match
+
+
+def negate(f: TextFilter) -> TextFilter:
+    """Return a filter that inverts the result of *f*."""
+
+    def _match(text: str) -> bool:
+        return not f(text)
+
+    return _match

scraper_framework/rate_limiter.py

@@ -0,0 +1,107 @@
+"""Rate limiting utilities for scrapers."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import time
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+
+@dataclass
+class RateLimiter:
+    """Configurable rate limiter with jittered sleep.
+
+    Supports both synchronous and asynchronous usage. The sleep duration is
+    uniformly sampled from ``[min_seconds, max_seconds]`` to avoid
+    thundering-herd patterns.
+
+    Example::
+
+        limiter = RateLimiter(min_seconds=1.0, max_seconds=3.0)
+        for url in urls:
+            fetch(url)
+            limiter.wait()
+
+    Async usage::
+
+        limiter = RateLimiter(min_seconds=0.5, max_seconds=1.5)
+        for url in urls:
+            await fetch(url)
+            await limiter.async_wait()
+    """
+
+    min_seconds: float = 2.0
+    max_seconds: float = 5.0
+    _request_count: int = field(default=0, init=False, repr=False)
+    _last_request_at: datetime | None = field(default=None, init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        if self.min_seconds < 0:
+            msg = f"min_seconds must be >= 0, got {self.min_seconds}"
+            raise ValueError(msg)
+        if self.max_seconds < self.min_seconds:
+            msg = f"max_seconds ({self.max_seconds}) must be >= min_seconds ({self.min_seconds})"
+            raise ValueError(msg)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    @property
+    def request_count(self) -> int:
+        """Number of waits performed."""
+        return self._request_count
+
+    @property
+    def last_request_at(self) -> datetime | None:
+        """Timestamp of the most recent wait."""
+        return self._last_request_at
+
+    def wait(self, override: tuple[float, float] | None = None) -> float:
+        """Block the current thread for a jittered duration.
+
+        Args:
+            override: Optional ``(min, max)`` tuple to use instead of the
+                instance defaults.
+
+        Returns:
+            The actual number of seconds slept.
+        """
+        duration = self._pick_duration(override)
+        time.sleep(duration)
+        self._record()
+        return duration
+
+    async def async_wait(self, override: tuple[float, float] | None = None) -> float:
+        """Async version of :meth:`wait`.
+
+        Args:
+            override: Optional ``(min, max)`` tuple to use instead of the
+                instance defaults.
+
+        Returns:
+            The actual number of seconds slept.
+        """
+        duration = self._pick_duration(override)
+        await asyncio.sleep(duration)
+        self._record()
+        return duration
+
+    def reset(self) -> None:
+        """Reset the internal counters."""
+        self._request_count = 0
+        self._last_request_at = None
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _pick_duration(self, override: tuple[float, float] | None) -> float:
+        lo, hi = override or (self.min_seconds, self.max_seconds)
+        return random.uniform(lo, hi)
+
+    def _record(self) -> None:
+        self._request_count += 1
+        self._last_request_at = datetime.now(UTC)

scraper_framework/registry.py

@@ -0,0 +1,119 @@
+"""Generic scraper registry and factory.
+
+The registry is a simple name-to-class mapping that supports both imperative
+registration and a decorator-based approach.
+
+Example::
+
+    from scraper_framework import BaseScraper, ScraperResult
+    from scraper_framework.registry import ScraperRegistry
+
+    registry = ScraperRegistry()
+
+    @registry.register("my-source")
+    class MyScraper(BaseScraper):
+        source = "my-source"
+        id_prefix = "ms"
+
+        def _run(self) -> ScraperResult:
+            ...
+
+    # Later
+    scraper = registry.get("my-source")
+    result = scraper.run()
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from scraper_framework.base import BaseScraper
+
+
+class ScraperRegistry:
+    """Thread-safe registry mapping scraper names to their classes.
+
+    Supports two registration styles:
+
+    1. **Imperative** -- ``registry.add("name", MyScraperClass)``
+    2. **Decorator** -- ``@registry.register("name")``
+
+    Retrieval always returns a *new instance* via :meth:`get`.
+    """
+
+    def __init__(self) -> None:
+        self._registry: dict[str, type[BaseScraper]] = {}
+
+    # ------------------------------------------------------------------
+    # Registration
+    # ------------------------------------------------------------------
+
+    def add(self, name: str, cls: type[BaseScraper]) -> None:
+        """Register a scraper class under *name*.
+
+        Args:
+            name: Unique identifier (e.g. ``"greenhouse"``).
+            cls: A concrete subclass of :class:`BaseScraper`.
+
+        Raises:
+            TypeError: If *cls* is not a subclass of ``BaseScraper``.
+            ValueError: If *name* is already registered.
+        """
+        if not (isinstance(cls, type) and issubclass(cls, BaseScraper)):
+            msg = f"Expected a BaseScraper subclass, got {cls!r}"
+            raise TypeError(msg)
+        if name in self._registry:
+            msg = f"Scraper '{name}' is already registered"
+            raise ValueError(msg)
+        self._registry[name] = cls
+
+    def register(self, name: str):
+        """Decorator that registers a scraper class.
+
+        Usage::
+
+            @registry.register("my-source")
+            class MyScraper(BaseScraper):
+                ...
+        """
+
+        def decorator[T: type[BaseScraper]](cls: T) -> T:
+            self.add(name, cls)  # type: ignore[arg-type]
+            return cls
+
+        return decorator
+
+    # ------------------------------------------------------------------
+    # Retrieval
+    # ------------------------------------------------------------------
+
+    def get(self, name: str, *, config: dict[str, Any] | None = None) -> BaseScraper:
+        """Instantiate and return a scraper by name.
+
+        Args:
+            name: The registered scraper name.
+            config: Optional config dict passed to the scraper constructor.
+
+        Raises:
+            KeyError: If *name* is not registered.
+        """
+        cls = self._registry.get(name)
+        if cls is None:
+            available = ", ".join(sorted(self._registry)) or "(none)"
+            msg = f"Unknown scraper: '{name}'. Available: {available}"
+            raise KeyError(msg)
+        return cls(config=config)
+
+    def list_names(self) -> list[str]:
+        """Return a sorted list of all registered scraper names."""
+        return sorted(self._registry)
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._registry
+
+    def __len__(self) -> int:
+        return len(self._registry)
+
+    def __repr__(self) -> str:
+        names = ", ".join(sorted(self._registry))
+        return f"ScraperRegistry([{names}])"

scraper_framework/result.py

@@ -0,0 +1,58 @@
+"""Scraper result tracking."""
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+
+@dataclass
+class ScraperResult:
+    """Accumulated result from a single scraper run.
+
+    Tracks fetch counts, insert counts, errors, and timing automatically.
+    Call ``finish()`` or rely on ``BaseScraper.run()`` to close the timer.
+
+    Example::
+
+        result = ScraperResult(source="my-source")
+        result.total_fetched += 10
+        result.new_inserted += 3
+        result.finish()
+        print(result.to_dict())
+    """
+
+    source: str
+    total_fetched: int = 0
+    new_inserted: int = 0
+    errors: int = 0
+    started_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    finished_at: datetime | None = None
+    metadata: dict[str, object] = field(default_factory=dict)
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def finish(self) -> None:
+        """Mark the run as finished with the current UTC timestamp."""
+        self.finished_at = datetime.now(UTC)
+
+    @property
+    def duration_seconds(self) -> float:
+        """Elapsed seconds between start and finish (or now if still running)."""
+        end = self.finished_at or datetime.now(UTC)
+        return round((end - self.started_at).total_seconds(), 1)
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a JSON-safe dictionary of the result."""
+        return {
+            "source": self.source,
+            "total_fetched": self.total_fetched,
+            "new_inserted": self.new_inserted,
+            "errors": self.errors,
+            "duration_seconds": self.duration_seconds,
+            "metadata": self.metadata,
+        }