self-healing-scraper 0.1.0__tar.gz

self_healing_scraper-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,18 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+      - run: uv sync --group dev
+      - run: uv run pre-commit run --all-files
+      - run: uv run pytest

self_healing_scraper-0.1.0/.gitignore

@@ -0,0 +1,19 @@
+.env
+__pycache__/
+.DS_Store
+.venv/
+.nimbella
+.deployed
+logs/
+
+# uv / tooling
+.ruff_cache/
+.pytest_cache/
+.mypy_cache/
+.ty_cache/
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+test_results/

self_healing_scraper-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+exclude: ^(\.cursor/|\.venv/)
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+        args: ["--maxkb=1024"]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.16.0
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/astral-sh/ty-pre-commit
+    rev: v0.0.63
+    hooks:
+      - id: ty

self_healing_scraper-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

self_healing_scraper-0.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: self-healing-scraper
+Version: 0.1.0
+Summary: Generic self-healing declarative web scraper engine
+Requires-Python: >=3.12
+Requires-Dist: beautifulsoup4>=4.15.0
+Requires-Dist: crawl4ai>=0.7.0
+Requires-Dist: openai>=2.48.0
+Requires-Dist: pydantic-settings>=2.14.2
+Requires-Dist: pydantic<3,>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: typing-extensions>=4.5.0

self_healing_scraper-0.1.0/README.md

@@ -0,0 +1,82 @@
+# self-healing-scraper
+
+Generic self-healing declarative web scraper. Pass a URL and a domain config; the engine looks up a stored parser by URL regex, creates one with AI if missing, executes CSS extractors, validates items, and repairs the parser when checks fail.
+
+Persistence is **not** included — inject a [`ParserStore`](src/self_healing_scraper/store.py) implementation (SQL, memory, etc.).
+
+## Install
+
+```bash
+# From GitHub
+uv add "self-healing-scraper @ git+https://github.com/SollalF/self-healing-scraper"
+
+# Local path (development)
+uv add --editable ../self-healing-scraper
+
+# Browser deps used by Crawl4AI
+uv run playwright install chromium
+# or: uv run crawl4ai-setup
+```
+
+## Quick example
+
+```python
+import asyncio
+from self_healing_scraper import scrape_url, ScrapeDomain, DomainPrompts
+from self_healing_scraper.store import InMemoryParserStore
+
+domain = ScrapeDomain(
+    prompts=DomainPrompts(
+        create_system="...",  # your domain prompts
+        create_user_template="...",
+        repair_system="...",
+        repair_user_template="...",
+    ),
+    default_required_fields=["title", "url"],
+)
+
+async def main() -> None:
+    store = InMemoryParserStore()
+    result = await scrape_url(
+        "https://example.com/list",
+        store=store,
+        domain=domain,
+    )
+    print(result.items)
+
+asyncio.run(main())
+```
+
+## Configuration
+
+| Env | Default | Purpose |
+|-----|---------|---------|
+| `LLM_API_KEY` | — | Required to create/repair parsers |
+| `LLM_MODEL` | `gpt-4o` | Model for parser agent |
+| `LLM_BASE_URL` | — | OpenAI-compatible API base |
+| `MAX_REPAIR_ATTEMPTS` | `3` | Self-heal loop limit |
+| `CRAWL_TIMEOUT_MS` | `30000` | Page load timeout |
+| `PAGE_SAMPLE_CHARS` | `12000` | HTML sample size sent to the AI |
+
+## Layout
+
+```
+src/self_healing_scraper/
+  scrape.py       # heal loop
+  store.py        # ParserStore Protocol + helpers
+  domain.py       # ScrapeDomain / DomainPrompts
+  fetch/          # Crawl4AI wrapper
+  runtime/        # executor + core validators
+  agent/          # create / repair LLM
+tests/
+```
+
+## Develop
+
+```bash
+uv sync
+uv run pre-commit install
+uv run pytest
+```
+
+CI runs `pre-commit` (ruff + ty + basic hooks) and pytest on every push/PR.

self_healing_scraper-0.1.0/pyproject.toml

@@ -0,0 +1,69 @@
+[project]
+name = "self-healing-scraper"
+version = "0.1.0"
+description = "Generic self-healing declarative web scraper engine"
+requires-python = ">=3.12"
+dependencies = [
+    "beautifulsoup4>=4.15.0",
+    "crawl4ai>=0.7.0",
+    "openai>=2.48.0",
+    "pydantic>=2.0.0,<3",
+    "pydantic-settings>=2.14.2",
+    "python-dotenv>=1.0.0",
+    "typing-extensions>=4.5.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/self_healing_scraper"]
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.6.1",
+    "pytest>=9.1.1",
+    "pytest-asyncio>=0.25.0",
+    "ruff>=0.16.0",
+    "ty>=0.0.63",
+]
+
+[tool.uv]
+package = true
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88
+src = ["src"]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "UP",
+    "B",
+    "C4",
+    "SIM",
+]
+ignore = [
+    "E501",
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["self_healing_scraper"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.ty.environment]
+root = ["./src"]
+python-version = "3.12"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"

self_healing_scraper-0.1.0/src/self_healing_scraper/__init__.py

@@ -0,0 +1,22 @@
+"""Generic self-healing declarative web scraper engine."""
+
+from self_healing_scraper.domain import DomainPrompts, ScrapeDomain
+from self_healing_scraper.models import ScrapeResult
+from self_healing_scraper.scrape import normalize_url, scrape_url, scrape_urls
+from self_healing_scraper.store import (
+    InMemoryParserStore,
+    ParserStore,
+    best_parser_match,
+)
+
+__all__ = [
+    "DomainPrompts",
+    "InMemoryParserStore",
+    "ParserStore",
+    "ScrapeDomain",
+    "ScrapeResult",
+    "best_parser_match",
+    "normalize_url",
+    "scrape_url",
+    "scrape_urls",
+]

self_healing_scraper-0.1.0/src/self_healing_scraper/agent/__init__.py

@@ -0,0 +1 @@
+"""LLM-assisted parser create / repair."""

self_healing_scraper-0.1.0/src/self_healing_scraper/agent/create_parser.py

@@ -0,0 +1,69 @@
+"""AI-assisted parser creation."""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from self_healing_scraper.agent.llm import complete_json
+from self_healing_scraper.agent.normalize import normalize_generated_payload
+from self_healing_scraper.domain import ScrapeDomain
+from self_healing_scraper.models import GeneratedParser, PageContent, PageKind
+from self_healing_scraper.settings import Settings, get_settings
+
+
+def _page_kind_hint(url: str) -> str:
+    path = urlparse(url).path.rstrip("/")
+    # Heuristic only — the model may override.
+    articleish = (
+        any(part.isdigit() and len(part) >= 4 for part in path.split("/") if part)
+        or path.count("/") >= 3
+    )
+    return PageKind.ARTICLE.value if articleish else PageKind.LISTING.value
+
+
+def _html_sample(html: str, limit: int) -> str:
+    """Prefer the main content region so chrome/nav does not eat the sample budget."""
+    lowered = html.lower()
+    for marker in ("<main", 'role="main"', 'id="content"', 'class="content"'):
+        idx = lowered.find(marker)
+        if idx != -1:
+            start = max(0, idx - 200)
+            return html[start : start + limit]
+    # Fall back to the middle/end of the document where listings often live.
+    if len(html) > limit:
+        return html[-(limit):]
+    return html
+
+
+async def create_parser(
+    page: PageContent,
+    domain: ScrapeDomain,
+    settings: Settings | None = None,
+) -> GeneratedParser:
+    cfg = settings or get_settings()
+    # Markdown from Crawl4AI is usually denser for listings than truncated HTML.
+    markdown = (page.markdown or "")[: cfg.page_sample_chars]
+    sample = _html_sample(page.html, min(cfg.page_sample_chars, 8000))
+    payload = await complete_json(
+        system=domain.prompts.create_system,
+        user=domain.prompts.create_user_template.format(
+            url=page.url,
+            page_kind_hint=_page_kind_hint(page.url),
+            html_sample=sample,
+            markdown_sample=markdown,
+        ),
+        settings=cfg,
+    )
+    return GeneratedParser.model_validate(
+        normalize_generated_payload(
+            payload,
+            known_checks=known_checks_for_domain(domain),
+            default_required_fields=domain.default_required_fields,
+        )
+    )
+
+
+def known_checks_for_domain(domain: ScrapeDomain) -> frozenset[str]:
+    from self_healing_scraper.runtime.validators import CORE_KNOWN_CHECKS
+
+    return CORE_KNOWN_CHECKS | domain.known_checks | frozenset(domain.extra_validators)

self_healing_scraper-0.1.0/src/self_healing_scraper/agent/llm.py

@@ -0,0 +1,77 @@
+"""Thin OpenAI-compatible LLM client wrapper."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from self_healing_scraper.settings import Settings, get_settings
+
+logger = logging.getLogger(__name__)
+
+KIMI_CODING_BASE_URL = "https://api.kimi.com/coding/v1"
+MOONSHOT_BASE_URL = "https://api.moonshot.ai/v1"
+
+
+def resolve_llm_base_url(api_key: str, configured: str) -> str | None:
+    """Pick the right OpenAI-compatible base URL for the key type."""
+    key = api_key.strip()
+    base = configured.strip()
+    if key.startswith("sk-kimi-"):
+        # Kimi Code keys are rejected by api.moonshot.ai
+        if not base or "moonshot.ai" in base:
+            return KIMI_CODING_BASE_URL
+        return base
+    return base or None
+
+
+def _default_headers(base_url: str | None) -> dict[str, str] | None:
+    if base_url and "api.kimi.com" in base_url:
+        # Coding endpoint gates access behind a recognized agent UA.
+        return {"User-Agent": "KimiCLI/1.0"}
+    return None
+
+
+async def complete_json(
+    *,
+    system: str,
+    user: str,
+    settings: Settings | None = None,
+) -> dict[str, Any]:
+    cfg = settings or get_settings()
+    if not cfg.llm_api_key:
+        raise RuntimeError("LLM_API_KEY is required to create or repair parsers")
+
+    from openai import AsyncOpenAI
+
+    base_url = resolve_llm_base_url(cfg.llm_api_key, cfg.llm_base_url)
+    headers = _default_headers(base_url)
+    logger.info("LLM request model=%s base_url=%s", cfg.llm_model, base_url)
+
+    client = AsyncOpenAI(
+        api_key=cfg.llm_api_key,
+        base_url=base_url,
+        default_headers=headers,
+    )
+    create_kwargs: dict[str, Any] = {
+        "model": cfg.llm_model,
+        "response_format": {"type": "json_object"},
+        "messages": [
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+    }
+    # Some models (e.g. kimi-k3) only accept temperature=1.
+    if "kimi" in cfg.llm_model.lower():
+        create_kwargs["temperature"] = 1
+    else:
+        create_kwargs["temperature"] = 0.2
+
+    response = await client.chat.completions.create(**create_kwargs)
+    content = response.choices[0].message.content or "{}"
+    logger.debug("LLM raw response: %s", content[:500])
+    data = json.loads(content)
+    if not isinstance(data, dict):
+        raise ValueError("LLM did not return a JSON object")
+    return data

self_healing_scraper-0.1.0/src/self_healing_scraper/agent/normalize.py

@@ -0,0 +1,59 @@
+"""Normalize LLM parser payloads before Pydantic validation."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from self_healing_scraper.runtime.validators import CORE_KNOWN_CHECKS
+
+
+def normalize_generated_payload(
+    payload: dict[str, Any],
+    *,
+    known_checks: frozenset[str] | None = None,
+    default_required_fields: list[str] | None = None,
+) -> dict[str, Any]:
+    """Drop null/invalid field extractors and coerce loose shapes from the LLM."""
+    allowed = known_checks if known_checks is not None else CORE_KNOWN_CHECKS
+    required = default_required_fields or ["title", "url"]
+    data = dict(payload)
+    definition = data.get("definition")
+    if isinstance(definition, dict):
+        fields = definition.get("fields")
+        if isinstance(fields, dict):
+            cleaned: dict[str, Any] = {}
+            for name, extractor in fields.items():
+                if extractor is None:
+                    continue
+                if isinstance(extractor, dict) and extractor.get("selector"):
+                    cleaned[name] = {
+                        "selector": extractor["selector"],
+                        "attr": extractor.get("attr") or "text",
+                        "many": bool(extractor.get("many", False)),
+                    }
+            definition = {**definition, "fields": cleaned}
+            data["definition"] = definition
+
+    validations = data.get("validations")
+    if isinstance(validations, dict):
+        checks = validations.get("checks")
+        if isinstance(checks, list):
+            kept = [
+                c for c in checks if isinstance(c, dict) and c.get("type") in allowed
+            ]
+            # Ensure a minimal useful suite remains.
+            types = {c["type"] for c in kept}
+            if "min_count" not in types:
+                kept.insert(0, {"type": "min_count", "value": 1})
+            if "required_fields" not in types:
+                kept.append({"type": "required_fields", "fields": list(required)})
+            data["validations"] = {**validations, "checks": kept}
+    elif validations is None:
+        data["validations"] = {
+            "checks": [
+                {"type": "min_count", "value": 1},
+                {"type": "required_fields", "fields": list(required)},
+            ]
+        }
+
+    return data

self_healing_scraper-0.1.0/src/self_healing_scraper/agent/repair_parser.py

@@ -0,0 +1,61 @@
+"""AI-assisted parser repair."""
+
+from __future__ import annotations
+
+import json
+
+from self_healing_scraper.agent.create_parser import known_checks_for_domain
+from self_healing_scraper.agent.llm import complete_json
+from self_healing_scraper.agent.normalize import normalize_generated_payload
+from self_healing_scraper.domain import ScrapeDomain
+from self_healing_scraper.models import (
+    GeneratedParser,
+    PageContent,
+    ParserDefinition,
+    ValidationResult,
+    ValidationSuite,
+)
+from self_healing_scraper.settings import Settings, get_settings
+
+
+async def repair_parser(
+    *,
+    page: PageContent,
+    name: str,
+    url_pattern: str,
+    page_kind: str,
+    definition: ParserDefinition,
+    validations: ValidationSuite,
+    validation_result: ValidationResult,
+    domain: ScrapeDomain,
+    settings: Settings | None = None,
+) -> GeneratedParser:
+    cfg = settings or get_settings()
+    current = {
+        "name": name,
+        "url_pattern": url_pattern,
+        "page_kind": page_kind,
+        "definition": definition.model_dump(),
+        "validations": validations.model_dump(),
+    }
+    payload = await complete_json(
+        system=domain.prompts.repair_system,
+        user=domain.prompts.repair_user_template.format(
+            url=page.url,
+            current_parser=json.dumps(current, indent=2),
+            failures=validation_result.model_dump_json(indent=2),
+            html_sample=page.html[: cfg.page_sample_chars],
+        ),
+        settings=cfg,
+    )
+    # Preserve identity fields if the model omits them.
+    payload.setdefault("name", name)
+    payload.setdefault("url_pattern", url_pattern)
+    payload.setdefault("page_kind", page_kind)
+    return GeneratedParser.model_validate(
+        normalize_generated_payload(
+            payload,
+            known_checks=known_checks_for_domain(domain),
+            default_required_fields=domain.default_required_fields,
+        )
+    )

self_healing_scraper-0.1.0/src/self_healing_scraper/asyncio_compat.py

@@ -0,0 +1,41 @@
+"""Windows-compatible asyncio helpers for Playwright (and consumers using psycopg)."""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from collections.abc import Callable, Coroutine
+from typing import cast
+
+
+def configure_event_loop_policy() -> None:
+    """Use SelectorEventLoop on Windows so psycopg async works in the main loop."""
+    if sys.platform == "win32":
+        # Still required for psycopg async on Windows under Python 3.12.
+        asyncio.set_event_loop_policy(  # ty: ignore[deprecated]
+            asyncio.WindowsSelectorEventLoopPolicy()
+        )
+
+
+def run[T](coro: Coroutine[object, object, T]) -> T:
+    configure_event_loop_policy()
+    return asyncio.run(coro)
+
+
+def _run_in_fresh_loop[T](coro: Coroutine[object, object, T]) -> T:
+    """Run a coroutine on a fresh loop (Proactor on Windows for Playwright)."""
+    if sys.platform == "win32":
+        loop = asyncio.ProactorEventLoop()
+    else:
+        loop = asyncio.new_event_loop()
+    try:
+        return loop.run_until_complete(coro)
+    finally:
+        loop.close()
+
+
+async def run_playwright[T](factory: Callable[[], Coroutine[object, object, T]]) -> T:
+    """Run Playwright/Crawl4AI work on a loop that supports subprocesses."""
+    if sys.platform == "win32":
+        return cast(T, await asyncio.to_thread(_run_in_fresh_loop, factory()))
+    return await factory()

self_healing_scraper-0.1.0/src/self_healing_scraper/domain.py

@@ -0,0 +1,37 @@
+"""Domain plug-in surface for prompts and extra validators."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from self_healing_scraper.models import (
+    PageContent,
+    ValidationCheck,
+    ValidationFailure,
+)
+
+CheckFn = Callable[
+    [ValidationCheck, list[dict[str, Any]], PageContent | None],
+    ValidationFailure | None,
+]
+
+
+@dataclass(frozen=True)
+class DomainPrompts:
+    create_system: str
+    create_user_template: str
+    repair_system: str
+    repair_user_template: str
+
+
+@dataclass(frozen=True)
+class ScrapeDomain:
+    """Specialization knobs for a scraping product (news, jobs, etc.)."""
+
+    prompts: DomainPrompts
+    default_required_fields: list[str] = field(default_factory=lambda: ["title", "url"])
+    known_checks: frozenset[str] = field(default_factory=frozenset)
+    extra_validators: dict[str, CheckFn] = field(default_factory=dict)
+    item_builder: Callable[[dict[str, Any]], Any] | None = None

self_healing_scraper-0.1.0/src/self_healing_scraper/fetch/__init__.py

@@ -0,0 +1,3 @@
+from self_healing_scraper.fetch.crawler import fetch_page
+
+__all__ = ["fetch_page"]