msaas-scraper-framework 0.1.0__tar.gz

msaas_scraper_framework-0.1.0/.gitignore

@@ -0,0 +1,21 @@
+node_modules/
+dist/
+.next/
+.turbo/
+*.pyc
+__pycache__/
+.venv/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.env
+.env.local
+.env.*.local
+.DS_Store
+coverage/
+
+# Runtime artifacts
+logs_llm/
+vectors.db
+vectors.db-shm
+vectors.db-wal

msaas_scraper_framework-0.1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: msaas-scraper-framework
+Version: 0.1.0
+Summary: Generic web scraper framework with registry, rate limiting, and composable filters
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: msaas-api-core
+Requires-Dist: msaas-errors
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'

msaas_scraper_framework-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "msaas-scraper-framework"
+version = "0.1.0"
+description = "Generic web scraper framework with registry, rate limiting, and composable filters"
+requires-python = ">=3.12"
+dependencies = [
+    "httpx>=0.27.0",
+    "msaas-errors",
+    "msaas-api-core",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+    "httpx>=0.27.0",
+    "ruff>=0.8",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scraper_framework"]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP", "B", "SIM", "TCH"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+
+[tool.uv.sources]
+msaas-api-core = { workspace = true }
+msaas-errors = { workspace = true }

msaas_scraper_framework-0.1.0/src/scraper_framework/__init__.py

@@ -0,0 +1,52 @@
+"""scraper-framework: Generic web scraper framework with registry, rate limiting, and filters.
+
+Usage::
+
+    from scraper_framework import BaseScraper, ScraperResult, ScraperRegistry
+
+    registry = ScraperRegistry()
+
+    @registry.register("example")
+    class ExampleScraper(BaseScraper):
+        source = "example"
+        id_prefix = "ex"
+
+        def _run(self) -> ScraperResult:
+            with self.make_client() as client:
+                resp = client.get("https://example.com/api")
+                self.result.total_fetched += len(resp.json())
+                self.sleep()
+            return self.result
+
+    scraper = registry.get("example")
+    result = scraper.run()
+    print(result.to_dict())
+"""
+
+from scraper_framework.base import BaseScraper
+from scraper_framework.filters import (
+    all_match,
+    any_match,
+    keyword_filter,
+    length_filter,
+    negate,
+    none_match,
+    regex_filter,
+)
+from scraper_framework.rate_limiter import RateLimiter
+from scraper_framework.registry import ScraperRegistry
+from scraper_framework.result import ScraperResult
+
+__all__ = [
+    "BaseScraper",
+    "RateLimiter",
+    "ScraperRegistry",
+    "ScraperResult",
+    "all_match",
+    "any_match",
+    "keyword_filter",
+    "length_filter",
+    "negate",
+    "none_match",
+    "regex_filter",
+]

msaas_scraper_framework-0.1.0/src/scraper_framework/base.py

@@ -0,0 +1,169 @@
+"""Generic base scraper class.
+
+Provides the abstract skeleton that concrete scrapers implement. Handles
+lifecycle (setup, run, teardown), HTTP client creation, rate limiting, and
+result tracking.
+
+Example subclass::
+
+    from scraper_framework import BaseScraper, ScraperResult
+
+    class MyScraper(BaseScraper):
+        source = "my-source"
+        id_prefix = "ms"
+
+        def _run(self) -> ScraperResult:
+            with self.make_client() as client:
+                resp = client.get("https://example.com/api/items")
+                items = resp.json()
+                self.result.total_fetched += len(items)
+                # ... process items ...
+                self.sleep()
+            return self.result
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from datetime import datetime, timezone
+from typing import Any
+
+import httpx
+
+from scraper_framework.rate_limiter import RateLimiter
+from scraper_framework.result import ScraperResult
+
+
+class BaseScraper(ABC):
+    """Abstract base class for all scrapers.
+
+    Subclasses **must** set ``source`` and implement :meth:`_run`.
+
+    Class attributes:
+        source: Identifier for this scraper (e.g. ``"greenhouse"``).
+        id_prefix: Short prefix for generated record IDs (e.g. ``"gh"``).
+        rate_limit_range: Default ``(min, max)`` seconds between requests.
+    """
+
+    source: str = ""
+    id_prefix: str = ""
+    rate_limit_range: tuple[float, float] = (2.0, 5.0)
+
+    def __init__(self, *, config: dict[str, Any] | None = None) -> None:
+        self.config: dict[str, Any] = config or {}
+        self.result = ScraperResult(source=self.source)
+        self.rate_limiter = RateLimiter(
+            min_seconds=self.rate_limit_range[0],
+            max_seconds=self.rate_limit_range[1],
+        )
+        try:
+            import structlog
+            self.log = structlog.get_logger(f"scraper.{self.source}").bind(source=self.source)
+        except ImportError:
+            self.log = logging.getLogger(f"scraper.{self.source}")
+
+    # ------------------------------------------------------------------
+    # Lifecycle hooks (override in subclasses if needed)
+    # ------------------------------------------------------------------
+
+    def setup(self) -> None:
+        """Called before :meth:`_run`. Override for one-time initialisation."""
+
+    def teardown(self) -> None:
+        """Called after :meth:`_run` completes (even on error). Override for cleanup."""
+
+    # ------------------------------------------------------------------
+    # Abstract
+    # ------------------------------------------------------------------
+
+    @abstractmethod
+    def _run(self) -> ScraperResult:
+        """Execute the scraping logic.
+
+        Must populate ``self.result`` and return it. Implementations should
+        call ``self.sleep()`` between HTTP requests to respect rate limits.
+        """
+        ...
+
+    # ------------------------------------------------------------------
+    # Public entry point
+    # ------------------------------------------------------------------
+
+    def run(
+        self,
+        *,
+        on_before: Callable[["BaseScraper"], None] | None = None,
+        on_after: Callable[["BaseScraper", ScraperResult], None] | None = None,
+    ) -> ScraperResult:
+        """Public entry point: setup, run scraper, teardown, finalize result.
+
+        Args:
+            on_before: Optional callback invoked after :meth:`setup` but
+                before :meth:`_run`.
+            on_after: Optional callback invoked after :meth:`_run` with the
+                result.
+
+        Returns:
+            The populated :class:`ScraperResult`.
+        """
+        self.setup()
+        if on_before:
+            on_before(self)
+        try:
+            self._run()
+        except Exception:
+            self.result.errors += 1
+            self.log.exception("Scraper %s failed", self.source)
+            raise
+        finally:
+            self.teardown()
+            self.result.finish()
+        if on_after:
+            on_after(self, self.result)
+        return self.result
+
+    # ------------------------------------------------------------------
+    # Utilities
+    # ------------------------------------------------------------------
+
+    def sleep(self, range_override: tuple[float, float] | None = None) -> float:
+        """Rate-limiting sleep between requests.
+
+        Args:
+            range_override: Optional ``(min, max)`` seconds to override the
+                default rate limit range.
+
+        Returns:
+            Actual seconds slept.
+        """
+        return self.rate_limiter.wait(override=range_override)
+
+    def make_client(self, **kwargs: Any) -> httpx.Client:
+        """Create an :class:`httpx.Client` with sensible defaults.
+
+        Defaults:
+            - ``timeout``: 30 seconds
+            - ``follow_redirects``: True
+
+        All keyword arguments are forwarded to ``httpx.Client``, overriding
+        the defaults.
+        """
+        defaults: dict[str, Any] = {"timeout": 30.0, "follow_redirects": True}
+        defaults.update(kwargs)
+        return httpx.Client(**defaults)
+
+    def make_async_client(self, **kwargs: Any) -> httpx.AsyncClient:
+        """Create an :class:`httpx.AsyncClient` with sensible defaults.
+
+        Same defaults as :meth:`make_client`.
+        """
+        defaults: dict[str, Any] = {"timeout": 30.0, "follow_redirects": True}
+        defaults.update(kwargs)
+        return httpx.AsyncClient(**defaults)
+
+    @staticmethod
+    def now_iso() -> str:
+        """Current UTC timestamp in ISO-8601 format."""
+        return datetime.now(timezone.utc).isoformat()

msaas_scraper_framework-0.1.0/src/scraper_framework/filters.py

@@ -0,0 +1,135 @@
+"""Composable filter functions for scraper pipelines.
+
+Filters are plain functions with the signature ``(text: str) -> bool`` so they
+can be combined freely with :func:`all_match`, :func:`any_match`, and
+:func:`none_match`.
+
+Example::
+
+    from scraper_framework.filters import (
+        regex_filter, keyword_filter, all_match, none_match,
+    )
+
+    is_senior = keyword_filter({"senior", "staff", "lead"})
+    not_excluded = none_match(keyword_filter({"intern", "junior"}))
+    has_python = regex_filter(r"\\bpython\\b")
+
+    title = "Senior Python Engineer"
+    if is_senior(title) and not_excluded(title) and has_python(title):
+        print("Relevant!")
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+
+type TextFilter = Callable[[str], bool]
+
+
+# ---------------------------------------------------------------------------
+# Primitive filter builders
+# ---------------------------------------------------------------------------
+
+
+def keyword_filter(
+    keywords: set[str],
+    *,
+    case_sensitive: bool = False,
+) -> TextFilter:
+    """Return a filter that matches if *any* keyword appears in the text.
+
+    Keywords are matched as substrings. For word-boundary matching use
+    :func:`regex_filter` instead.
+
+    Args:
+        keywords: Set of keywords to look for.
+        case_sensitive: Whether matching is case-sensitive.
+    """
+    if not case_sensitive:
+        normalised = {k.lower() for k in keywords}
+
+        def _match(text: str) -> bool:
+            lowered = text.lower()
+            return any(kw in lowered for kw in normalised)
+    else:
+
+        def _match(text: str) -> bool:
+            return any(kw in text for kw in keywords)
+
+    return _match
+
+
+def regex_filter(pattern: str, *, flags: int = re.IGNORECASE) -> TextFilter:
+    """Return a filter that matches if the compiled regex finds a match.
+
+    Args:
+        pattern: A regular expression string.
+        flags: Regex flags (default: ``re.IGNORECASE``).
+    """
+    compiled = re.compile(pattern, flags)
+
+    def _match(text: str) -> bool:
+        return compiled.search(text) is not None
+
+    return _match
+
+
+def length_filter(*, min_length: int = 0, max_length: int | None = None) -> TextFilter:
+    """Return a filter that checks text length.
+
+    Args:
+        min_length: Minimum number of characters.
+        max_length: Maximum number of characters (``None`` for unlimited).
+    """
+
+    def _match(text: str) -> bool:
+        n = len(text)
+        if n < min_length:
+            return False
+        if max_length is not None and n > max_length:
+            return False
+        return True
+
+    return _match
+
+
+# ---------------------------------------------------------------------------
+# Combinators
+# ---------------------------------------------------------------------------
+
+
+def all_match(*filters: TextFilter) -> TextFilter:
+    """Return a filter that passes only if **all** sub-filters pass."""
+
+    def _match(text: str) -> bool:
+        return all(f(text) for f in filters)
+
+    return _match
+
+
+def any_match(*filters: TextFilter) -> TextFilter:
+    """Return a filter that passes if **any** sub-filter passes."""
+
+    def _match(text: str) -> bool:
+        return any(f(text) for f in filters)
+
+    return _match
+
+
+def none_match(*filters: TextFilter) -> TextFilter:
+    """Return a filter that passes only if **no** sub-filter passes."""
+
+    def _match(text: str) -> bool:
+        return not any(f(text) for f in filters)
+
+    return _match
+
+
+def negate(f: TextFilter) -> TextFilter:
+    """Return a filter that inverts the result of *f*."""
+
+    def _match(text: str) -> bool:
+        return not f(text)
+
+    return _match

msaas_scraper_framework-0.1.0/src/scraper_framework/rate_limiter.py

@@ -0,0 +1,110 @@
+"""Rate limiting utilities for scrapers."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+
+@dataclass
+class RateLimiter:
+    """Configurable rate limiter with jittered sleep.
+
+    Supports both synchronous and asynchronous usage. The sleep duration is
+    uniformly sampled from ``[min_seconds, max_seconds]`` to avoid
+    thundering-herd patterns.
+
+    Example::
+
+        limiter = RateLimiter(min_seconds=1.0, max_seconds=3.0)
+        for url in urls:
+            fetch(url)
+            limiter.wait()
+
+    Async usage::
+
+        limiter = RateLimiter(min_seconds=0.5, max_seconds=1.5)
+        for url in urls:
+            await fetch(url)
+            await limiter.async_wait()
+    """
+
+    min_seconds: float = 2.0
+    max_seconds: float = 5.0
+    _request_count: int = field(default=0, init=False, repr=False)
+    _last_request_at: datetime | None = field(default=None, init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        if self.min_seconds < 0:
+            msg = f"min_seconds must be >= 0, got {self.min_seconds}"
+            raise ValueError(msg)
+        if self.max_seconds < self.min_seconds:
+            msg = (
+                f"max_seconds ({self.max_seconds}) must be >= "
+                f"min_seconds ({self.min_seconds})"
+            )
+            raise ValueError(msg)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    @property
+    def request_count(self) -> int:
+        """Number of waits performed."""
+        return self._request_count
+
+    @property
+    def last_request_at(self) -> datetime | None:
+        """Timestamp of the most recent wait."""
+        return self._last_request_at
+
+    def wait(self, override: tuple[float, float] | None = None) -> float:
+        """Block the current thread for a jittered duration.
+
+        Args:
+            override: Optional ``(min, max)`` tuple to use instead of the
+                instance defaults.
+
+        Returns:
+            The actual number of seconds slept.
+        """
+        duration = self._pick_duration(override)
+        time.sleep(duration)
+        self._record()
+        return duration
+
+    async def async_wait(self, override: tuple[float, float] | None = None) -> float:
+        """Async version of :meth:`wait`.
+
+        Args:
+            override: Optional ``(min, max)`` tuple to use instead of the
+                instance defaults.
+
+        Returns:
+            The actual number of seconds slept.
+        """
+        duration = self._pick_duration(override)
+        await asyncio.sleep(duration)
+        self._record()
+        return duration
+
+    def reset(self) -> None:
+        """Reset the internal counters."""
+        self._request_count = 0
+        self._last_request_at = None
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _pick_duration(self, override: tuple[float, float] | None) -> float:
+        lo, hi = override or (self.min_seconds, self.max_seconds)
+        return random.uniform(lo, hi)
+
+    def _record(self) -> None:
+        self._request_count += 1
+        self._last_request_at = datetime.now(timezone.utc)

msaas_scraper_framework-0.1.0/src/scraper_framework/registry.py

@@ -0,0 +1,119 @@
+"""Generic scraper registry and factory.
+
+The registry is a simple name-to-class mapping that supports both imperative
+registration and a decorator-based approach.
+
+Example::
+
+    from scraper_framework import BaseScraper, ScraperResult
+    from scraper_framework.registry import ScraperRegistry
+
+    registry = ScraperRegistry()
+
+    @registry.register("my-source")
+    class MyScraper(BaseScraper):
+        source = "my-source"
+        id_prefix = "ms"
+
+        def _run(self) -> ScraperResult:
+            ...
+
+    # Later
+    scraper = registry.get("my-source")
+    result = scraper.run()
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from scraper_framework.base import BaseScraper
+
+
+class ScraperRegistry:
+    """Thread-safe registry mapping scraper names to their classes.
+
+    Supports two registration styles:
+
+    1. **Imperative** -- ``registry.add("name", MyScraperClass)``
+    2. **Decorator** -- ``@registry.register("name")``
+
+    Retrieval always returns a *new instance* via :meth:`get`.
+    """
+
+    def __init__(self) -> None:
+        self._registry: dict[str, type[BaseScraper]] = {}
+
+    # ------------------------------------------------------------------
+    # Registration
+    # ------------------------------------------------------------------
+
+    def add(self, name: str, cls: type[BaseScraper]) -> None:
+        """Register a scraper class under *name*.
+
+        Args:
+            name: Unique identifier (e.g. ``"greenhouse"``).
+            cls: A concrete subclass of :class:`BaseScraper`.
+
+        Raises:
+            TypeError: If *cls* is not a subclass of ``BaseScraper``.
+            ValueError: If *name* is already registered.
+        """
+        if not (isinstance(cls, type) and issubclass(cls, BaseScraper)):
+            msg = f"Expected a BaseScraper subclass, got {cls!r}"
+            raise TypeError(msg)
+        if name in self._registry:
+            msg = f"Scraper '{name}' is already registered"
+            raise ValueError(msg)
+        self._registry[name] = cls
+
+    def register(self, name: str):
+        """Decorator that registers a scraper class.
+
+        Usage::
+
+            @registry.register("my-source")
+            class MyScraper(BaseScraper):
+                ...
+        """
+
+        def decorator[T: type[BaseScraper]](cls: T) -> T:
+            self.add(name, cls)  # type: ignore[arg-type]
+            return cls
+
+        return decorator
+
+    # ------------------------------------------------------------------
+    # Retrieval
+    # ------------------------------------------------------------------
+
+    def get(self, name: str, *, config: dict[str, Any] | None = None) -> BaseScraper:
+        """Instantiate and return a scraper by name.
+
+        Args:
+            name: The registered scraper name.
+            config: Optional config dict passed to the scraper constructor.
+
+        Raises:
+            KeyError: If *name* is not registered.
+        """
+        cls = self._registry.get(name)
+        if cls is None:
+            available = ", ".join(sorted(self._registry)) or "(none)"
+            msg = f"Unknown scraper: '{name}'. Available: {available}"
+            raise KeyError(msg)
+        return cls(config=config)
+
+    def list_names(self) -> list[str]:
+        """Return a sorted list of all registered scraper names."""
+        return sorted(self._registry)
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._registry
+
+    def __len__(self) -> int:
+        return len(self._registry)
+
+    def __repr__(self) -> str:
+        names = ", ".join(sorted(self._registry))
+        return f"ScraperRegistry([{names}])"

msaas_scraper_framework-0.1.0/src/scraper_framework/result.py

@@ -0,0 +1,58 @@
+"""Scraper result tracking."""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+
+@dataclass
+class ScraperResult:
+    """Accumulated result from a single scraper run.
+
+    Tracks fetch counts, insert counts, errors, and timing automatically.
+    Call ``finish()`` or rely on ``BaseScraper.run()`` to close the timer.
+
+    Example::
+
+        result = ScraperResult(source="my-source")
+        result.total_fetched += 10
+        result.new_inserted += 3
+        result.finish()
+        print(result.to_dict())
+    """
+
+    source: str
+    total_fetched: int = 0
+    new_inserted: int = 0
+    errors: int = 0
+    started_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    finished_at: datetime | None = None
+    metadata: dict[str, object] = field(default_factory=dict)
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def finish(self) -> None:
+        """Mark the run as finished with the current UTC timestamp."""
+        self.finished_at = datetime.now(timezone.utc)
+
+    @property
+    def duration_seconds(self) -> float:
+        """Elapsed seconds between start and finish (or now if still running)."""
+        end = self.finished_at or datetime.now(timezone.utc)
+        return round((end - self.started_at).total_seconds(), 1)
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a JSON-safe dictionary of the result."""
+        return {
+            "source": self.source,
+            "total_fetched": self.total_fetched,
+            "new_inserted": self.new_inserted,
+            "errors": self.errors,
+            "duration_seconds": self.duration_seconds,
+            "metadata": self.metadata,
+        }

msaas_scraper_framework-0.1.0/tests/test_base.py

@@ -0,0 +1,418 @@
+"""Tests for the scraper framework core components."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from scraper_framework import (
+    BaseScraper,
+    RateLimiter,
+    ScraperRegistry,
+    ScraperResult,
+    all_match,
+    any_match,
+    keyword_filter,
+    length_filter,
+    negate,
+    none_match,
+    regex_filter,
+)
+
+
+# =========================================================================
+# Fixtures
+# =========================================================================
+
+
+class StubScraper(BaseScraper):
+    """Minimal concrete scraper for testing."""
+
+    source = "stub"
+    id_prefix = "st"
+    rate_limit_range = (0.0, 0.0)  # No delay in tests
+
+    def _run(self) -> ScraperResult:
+        self.result.total_fetched = 5
+        self.result.new_inserted = 3
+        return self.result
+
+
+class FailingScraper(BaseScraper):
+    """Scraper that raises during _run."""
+
+    source = "failing"
+    id_prefix = "fl"
+    rate_limit_range = (0.0, 0.0)
+
+    def _run(self) -> ScraperResult:
+        msg = "connection refused"
+        raise ConnectionError(msg)
+
+
+# =========================================================================
+# ScraperResult
+# =========================================================================
+
+
+class TestScraperResult:
+    def test_defaults(self) -> None:
+        result = ScraperResult(source="test")
+        assert result.source == "test"
+        assert result.total_fetched == 0
+        assert result.new_inserted == 0
+        assert result.errors == 0
+        assert result.finished_at is None
+        assert isinstance(result.started_at, datetime)
+
+    def test_finish(self) -> None:
+        result = ScraperResult(source="test")
+        result.finish()
+        assert result.finished_at is not None
+        assert result.finished_at >= result.started_at
+
+    def test_duration_seconds_running(self) -> None:
+        result = ScraperResult(source="test")
+        duration = result.duration_seconds
+        assert duration >= 0.0
+
+    def test_duration_seconds_finished(self) -> None:
+        result = ScraperResult(source="test")
+        result.finish()
+        duration = result.duration_seconds
+        assert duration >= 0.0
+
+    def test_to_dict(self) -> None:
+        result = ScraperResult(source="test")
+        result.total_fetched = 10
+        result.new_inserted = 4
+        result.errors = 1
+        result.metadata["custom"] = "value"
+        result.finish()
+
+        d = result.to_dict()
+        assert d["source"] == "test"
+        assert d["total_fetched"] == 10
+        assert d["new_inserted"] == 4
+        assert d["errors"] == 1
+        assert isinstance(d["duration_seconds"], float)
+        assert d["metadata"] == {"custom": "value"}
+
+    def test_metadata_isolation(self) -> None:
+        r1 = ScraperResult(source="a")
+        r2 = ScraperResult(source="b")
+        r1.metadata["key"] = 1
+        assert "key" not in r2.metadata
+
+
+# =========================================================================
+# BaseScraper
+# =========================================================================
+
+
+class TestBaseScraper:
+    def test_run_lifecycle(self) -> None:
+        scraper = StubScraper()
+        result = scraper.run()
+        assert result.total_fetched == 5
+        assert result.new_inserted == 3
+        assert result.finished_at is not None
+
+    def test_run_with_config(self) -> None:
+        scraper = StubScraper(config={"key": "value"})
+        assert scraper.config == {"key": "value"}
+
+    def test_run_calls_setup_and_teardown(self) -> None:
+        scraper = StubScraper()
+        scraper.setup = MagicMock()  # type: ignore[method-assign]
+        scraper.teardown = MagicMock()  # type: ignore[method-assign]
+
+        scraper.run()
+
+        scraper.setup.assert_called_once()
+        scraper.teardown.assert_called_once()
+
+    def test_run_on_before_callback(self) -> None:
+        scraper = StubScraper()
+        callback = MagicMock()
+
+        scraper.run(on_before=callback)
+
+        callback.assert_called_once_with(scraper)
+
+    def test_run_on_after_callback(self) -> None:
+        scraper = StubScraper()
+        callback = MagicMock()
+
+        result = scraper.run(on_after=callback)
+
+        callback.assert_called_once_with(scraper, result)
+
+    def test_failing_scraper_increments_errors_and_raises(self) -> None:
+        scraper = FailingScraper()
+        with pytest.raises(ConnectionError, match="connection refused"):
+            scraper.run()
+        assert scraper.result.errors == 1
+        # teardown is still called and result is finished
+        assert scraper.result.finished_at is not None
+
+    def test_teardown_called_on_failure(self) -> None:
+        scraper = FailingScraper()
+        scraper.teardown = MagicMock()  # type: ignore[method-assign]
+
+        with pytest.raises(ConnectionError):
+            scraper.run()
+
+        scraper.teardown.assert_called_once()
+
+    def test_sleep_delegates_to_rate_limiter(self) -> None:
+        scraper = StubScraper()
+        with patch.object(scraper.rate_limiter, "wait", return_value=0.0) as mock_wait:
+            scraper.sleep()
+            mock_wait.assert_called_once_with(override=None)
+
+    def test_sleep_with_override(self) -> None:
+        scraper = StubScraper()
+        with patch.object(scraper.rate_limiter, "wait", return_value=0.0) as mock_wait:
+            scraper.sleep(range_override=(1.0, 2.0))
+            mock_wait.assert_called_once_with(override=(1.0, 2.0))
+
+    def test_make_client(self) -> None:
+        scraper = StubScraper()
+        client = scraper.make_client()
+        assert client.timeout.read == 30.0
+        assert client.follow_redirects is True
+        client.close()
+
+    def test_make_client_custom_timeout(self) -> None:
+        scraper = StubScraper()
+        client = scraper.make_client(timeout=60.0)
+        assert client.timeout.read == 60.0
+        client.close()
+
+    def test_make_async_client(self) -> None:
+        scraper = StubScraper()
+        client = scraper.make_async_client()
+        assert client.timeout.read == 30.0
+        assert client.follow_redirects is True
+
+    def test_now_iso(self) -> None:
+        ts = StubScraper.now_iso()
+        parsed = datetime.fromisoformat(ts)
+        assert parsed.tzinfo == timezone.utc
+
+    def test_logger_name(self) -> None:
+        scraper = StubScraper()
+        # structlog loggers don't expose .name; check the logger works instead
+        try:
+            assert scraper.log.name == "scraper.stub"
+        except AttributeError:
+            # structlog BoundLogger — verify it's functional
+            assert scraper.log is not None
+
+
+# =========================================================================
+# RateLimiter
+# =========================================================================
+
+
+class TestRateLimiter:
+    def test_defaults(self) -> None:
+        limiter = RateLimiter()
+        assert limiter.min_seconds == 2.0
+        assert limiter.max_seconds == 5.0
+        assert limiter.request_count == 0
+        assert limiter.last_request_at is None
+
+    def test_wait_records_count(self) -> None:
+        limiter = RateLimiter(min_seconds=0.0, max_seconds=0.0)
+        limiter.wait()
+        limiter.wait()
+        assert limiter.request_count == 2
+        assert limiter.last_request_at is not None
+
+    def test_wait_with_override(self) -> None:
+        limiter = RateLimiter(min_seconds=10.0, max_seconds=20.0)
+        duration = limiter.wait(override=(0.0, 0.0))
+        assert duration == 0.0
+
+    def test_reset(self) -> None:
+        limiter = RateLimiter(min_seconds=0.0, max_seconds=0.0)
+        limiter.wait()
+        limiter.reset()
+        assert limiter.request_count == 0
+        assert limiter.last_request_at is None
+
+    def test_invalid_min_seconds(self) -> None:
+        with pytest.raises(ValueError, match="min_seconds must be >= 0"):
+            RateLimiter(min_seconds=-1.0)
+
+    def test_max_less_than_min(self) -> None:
+        with pytest.raises(ValueError, match="max_seconds"):
+            RateLimiter(min_seconds=5.0, max_seconds=1.0)
+
+    @pytest.mark.asyncio
+    async def test_async_wait(self) -> None:
+        limiter = RateLimiter(min_seconds=0.0, max_seconds=0.0)
+        duration = await limiter.async_wait()
+        assert duration == 0.0
+        assert limiter.request_count == 1
+
+
+# =========================================================================
+# ScraperRegistry
+# =========================================================================
+
+
+class TestScraperRegistry:
+    def test_add_and_get(self) -> None:
+        registry = ScraperRegistry()
+        registry.add("stub", StubScraper)
+        scraper = registry.get("stub")
+        assert isinstance(scraper, StubScraper)
+
+    def test_get_with_config(self) -> None:
+        registry = ScraperRegistry()
+        registry.add("stub", StubScraper)
+        scraper = registry.get("stub", config={"key": "val"})
+        assert scraper.config == {"key": "val"}
+
+    def test_register_decorator(self) -> None:
+        registry = ScraperRegistry()
+
+        @registry.register("decorated")
+        class DecoratedScraper(StubScraper):
+            source = "decorated"
+
+        assert "decorated" in registry
+        scraper = registry.get("decorated")
+        assert isinstance(scraper, DecoratedScraper)
+
+    def test_get_unknown_raises(self) -> None:
+        registry = ScraperRegistry()
+        with pytest.raises(KeyError, match="Unknown scraper: 'nope'"):
+            registry.get("nope")
+
+    def test_duplicate_add_raises(self) -> None:
+        registry = ScraperRegistry()
+        registry.add("stub", StubScraper)
+        with pytest.raises(ValueError, match="already registered"):
+            registry.add("stub", StubScraper)
+
+    def test_add_non_scraper_raises(self) -> None:
+        registry = ScraperRegistry()
+        with pytest.raises(TypeError, match="Expected a BaseScraper subclass"):
+            registry.add("bad", dict)  # type: ignore[arg-type]
+
+    def test_list_names(self) -> None:
+        registry = ScraperRegistry()
+        registry.add("beta", StubScraper)
+
+        class AnotherScraper(StubScraper):
+            source = "alpha"
+
+        registry.add("alpha", AnotherScraper)
+        assert registry.list_names() == ["alpha", "beta"]
+
+    def test_contains(self) -> None:
+        registry = ScraperRegistry()
+        registry.add("stub", StubScraper)
+        assert "stub" in registry
+        assert "missing" not in registry
+
+    def test_len(self) -> None:
+        registry = ScraperRegistry()
+        assert len(registry) == 0
+        registry.add("stub", StubScraper)
+        assert len(registry) == 1
+
+    def test_repr(self) -> None:
+        registry = ScraperRegistry()
+        registry.add("b", StubScraper)
+
+        class A(StubScraper):
+            source = "a"
+
+        registry.add("a", A)
+        assert repr(registry) == "ScraperRegistry([a, b])"
+
+
+# =========================================================================
+# Filters
+# =========================================================================
+
+
+class TestFilters:
+    def test_keyword_filter_case_insensitive(self) -> None:
+        f = keyword_filter({"senior", "staff"})
+        assert f("Senior Engineer") is True
+        assert f("STAFF developer") is True
+        assert f("Junior Developer") is False
+
+    def test_keyword_filter_case_sensitive(self) -> None:
+        f = keyword_filter({"Senior"}, case_sensitive=True)
+        assert f("Senior Engineer") is True
+        assert f("senior engineer") is False
+
+    def test_regex_filter(self) -> None:
+        f = regex_filter(r"\bpython\b")
+        assert f("Python Developer") is True
+        assert f("pythonic code") is False  # word boundary
+        assert f("Java Engineer") is False
+
+    def test_length_filter(self) -> None:
+        f = length_filter(min_length=3, max_length=10)
+        assert f("ab") is False
+        assert f("abc") is True
+        assert f("0123456789") is True
+        assert f("01234567890") is False
+
+    def test_length_filter_no_max(self) -> None:
+        f = length_filter(min_length=1)
+        assert f("") is False
+        assert f("x" * 10000) is True
+
+    def test_all_match(self) -> None:
+        f = all_match(
+            keyword_filter({"senior"}),
+            regex_filter(r"\bpython\b"),
+        )
+        assert f("Senior Python Developer") is True
+        assert f("Senior Java Developer") is False
+        assert f("Junior Python Developer") is False
+
+    def test_any_match(self) -> None:
+        f = any_match(
+            keyword_filter({"python"}),
+            keyword_filter({"rust"}),
+        )
+        assert f("Python Dev") is True
+        assert f("Rust Dev") is True
+        assert f("Java Dev") is False
+
+    def test_none_match(self) -> None:
+        f = none_match(keyword_filter({"intern", "junior"}))
+        assert f("Senior Engineer") is True
+        assert f("Junior Developer") is False
+        assert f("Intern position") is False
+
+    def test_negate(self) -> None:
+        f = negate(keyword_filter({"intern"}))
+        assert f("Senior Engineer") is True
+        assert f("Summer Intern") is False
+
+    def test_composable_pipeline(self) -> None:
+        """Full pipeline: senior + skills, no exclusions."""
+        is_senior = keyword_filter({"senior", "staff", "lead"})
+        not_excluded = none_match(keyword_filter({"intern", "junior"}))
+        has_tech = regex_filter(r"\b(python|typescript|react)\b")
+
+        pipeline = all_match(is_senior, not_excluded, has_tech)
+
+        assert pipeline("Senior Python Engineer") is True
+        assert pipeline("Lead React Developer") is True
+        assert pipeline("Junior Python Developer") is False
+        assert pipeline("Senior Marketing Manager") is False
+        assert pipeline("Staff Intern Program") is False