scrapemend 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scrapemend/__init__.py ADDED
@@ -0,0 +1,14 @@
1
+ """scrapemend — self-healing web scraping for Python.
2
+
3
+ Describe the data you want once. scrapemend extracts it with cached
4
+ selectors; when a site changes its markup and those selectors stop
5
+ matching, a language model inspects the new page and repairs them
6
+ automatically, then caches the fix so the next run is free again.
7
+ """
8
+
9
+ from scrapemend.result import ExtractionResult
10
+ from scrapemend.schema import Field, Schema
11
+ from scrapemend.scraper import Scraper
12
+
13
+ __all__ = ["ExtractionResult", "Field", "Schema", "Scraper"]
14
+ __version__ = "0.0.1"
scrapemend/cache.py ADDED
@@ -0,0 +1,57 @@
1
+ """On-disk selector cache keyed by (schema, site).
2
+
3
+ The cache is what makes the common path free: once selectors are known
4
+ for a given schema and host, extraction is pure parsing with no LLM call.
5
+ Healing only writes here when the cached selectors stop matching.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from pathlib import Path
12
+ from urllib.parse import urlparse
13
+
14
+
15
+ class SelectorCache:
16
+ """Maps a schema+host to the CSS selectors last known to work.
17
+
18
+ Stored as a single JSON file so it is trivial to inspect, diff, and
19
+ commit to a repo if a team wants to share healed selectors. (Shared,
20
+ hosted cache sync is a candidate Pro feature later.)
21
+ """
22
+
23
+ def __init__(self, path: str | Path = ".scrapemend-cache.json") -> None:
24
+ self.path = Path(path)
25
+ self._data: dict[str, dict[str, str]] = {}
26
+ self._load()
27
+
28
+ def _load(self) -> None:
29
+ if self.path.exists():
30
+ try:
31
+ self._data = json.loads(self.path.read_text(encoding="utf-8"))
32
+ except (json.JSONDecodeError, OSError):
33
+ self._data = {}
34
+
35
+ def _flush(self) -> None:
36
+ self.path.write_text(
37
+ json.dumps(self._data, indent=2, ensure_ascii=False), encoding="utf-8"
38
+ )
39
+
40
+ @staticmethod
41
+ def _key(schema_name: str, url: str) -> str:
42
+ host = urlparse(url).netloc or "unknown"
43
+ return f"{schema_name}@{host}"
44
+
45
+ def get(self, schema_name: str, url: str) -> dict[str, str] | None:
46
+ """Return cached {field_name: css_selector} or None if unknown."""
47
+ return self._data.get(self._key(schema_name, url))
48
+
49
+ def put(self, schema_name: str, url: str, selectors: dict[str, str]) -> None:
50
+ """Store healed selectors for this schema+host and persist."""
51
+ self._data[self._key(schema_name, url)] = selectors
52
+ self._flush()
53
+
54
+ def drop(self, schema_name: str, url: str) -> None:
55
+ """Forget selectors for a schema+host (forces a re-heal next run)."""
56
+ self._data.pop(self._key(schema_name, url), None)
57
+ self._flush()
scrapemend/condense.py ADDED
@@ -0,0 +1,60 @@
1
+ """Shrink HTML before it goes into a healing prompt.
2
+
3
+ The model only needs the page's *structure* to pick CSS selectors, not
4
+ its scripts, styles, or inline data payloads. Stripping those cuts prompt
5
+ tokens sharply (cheaper healing) and removes noise that can mislead the
6
+ model (more accurate selectors). This directly serves the library's
7
+ "robust but cheap" promise.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import re
13
+
14
+ from selectolax.lexbor import LexborHTMLParser
15
+
16
+ #: Tags that carry no structural signal for selector discovery.
17
+ _DROP_TAGS = ("script", "style", "noscript", "template", "svg", "iframe")
18
+
19
+ #: The only attributes that matter for building CSS selectors.
20
+ _KEEP_ATTRS = ("class", "id")
21
+
22
+ _WS = re.compile(r"\s+")
23
+
24
+
25
+ def condense(html: str, limit: int = 6000) -> str:
26
+ """Return a stripped-down version of ``html`` for prompting.
27
+
28
+ Removes non-structural tags, drops every attribute except class/id,
29
+ collapses whitespace, and finally caps length as a hard token guard.
30
+
31
+ Falls back to a plain truncation if parsing fails for any reason, so
32
+ healing never breaks just because a page is malformed.
33
+ """
34
+ try:
35
+ tree = LexborHTMLParser(html)
36
+ except Exception: # noqa: BLE001 # pragma: no cover - last-resort guard; never let healing crash on a weird page
37
+ return _WS.sub(" ", html)[:limit]
38
+
39
+ for tag in _DROP_TAGS:
40
+ for node in tree.css(tag):
41
+ node.decompose()
42
+
43
+ body = tree.body if tree.body is not None else tree.root
44
+ if body is None:
45
+ return _WS.sub(" ", html)[:limit]
46
+
47
+ for node in body.traverse(include_text=False):
48
+ attrs = getattr(node, "attributes", None)
49
+ if not attrs:
50
+ continue
51
+ for name in list(attrs.keys()):
52
+ if name not in _KEEP_ATTRS:
53
+ try:
54
+ del node.attrs[name]
55
+ except (KeyError, TypeError): # pragma: no cover - defensive
56
+ pass
57
+
58
+ cleaned = body.html or ""
59
+ cleaned = _WS.sub(" ", cleaned).strip()
60
+ return cleaned[:limit]
scrapemend/healer.py ADDED
@@ -0,0 +1,49 @@
1
+ """Selector discovery/repair via the LLM.
2
+
3
+ Given a page and a schema, ask the model for one CSS selector per field.
4
+ The healer is only invoked when cached selectors are missing or have
5
+ stopped matching, so its token cost is paid rarely, not per request.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+
12
+ from scrapemend.condense import condense
13
+ from scrapemend.llm import LLMClient
14
+ from scrapemend.schema import Schema
15
+
16
+ _SYSTEM = (
17
+ "You map data fields to CSS selectors for a web page. "
18
+ "Return ONLY a JSON object mapping each field name to a single CSS "
19
+ "selector string that selects the element holding that field's value. "
20
+ "No prose, no code fences."
21
+ )
22
+
23
+
24
+ def build_prompt(schema: Schema, html: str) -> str:
25
+ lines = [f"- {f.name}: {f.describe}" for f in schema.fields]
26
+ return (
27
+ "Fields to locate:\n"
28
+ + "\n".join(lines)
29
+ + "\n\nPage HTML (condensed, possibly truncated):\n"
30
+ + condense(html)
31
+ )
32
+
33
+
34
+ def heal(schema: Schema, html: str, client: LLMClient) -> dict[str, str]:
35
+ """Ask the LLM for fresh selectors. Returns {field_name: selector}.
36
+
37
+ Unknown or malformed replies yield an empty mapping rather than
38
+ raising, so the caller can record a clean failure and move on.
39
+ """
40
+ reply = client.complete(_SYSTEM, build_prompt(schema, html))
41
+ try:
42
+ parsed = json.loads(reply)
43
+ except json.JSONDecodeError:
44
+ return {}
45
+ if not isinstance(parsed, dict):
46
+ return {}
47
+ # Keep only known fields with string selectors.
48
+ valid = set(schema.field_names())
49
+ return {k: v for k, v in parsed.items() if k in valid and isinstance(v, str)}
scrapemend/llm.py ADDED
@@ -0,0 +1,60 @@
1
+ """The shared LLM layer.
2
+
3
+ This module is deliberately generic: a provider-agnostic client plus a
4
+ token/cost ledger. scrapemend uses it only for selector healing, but it
5
+ carries no scraping concepts, so the planned LLM-developer-tooling product
6
+ can reuse it wholesale. Keep it free of any scraping imports.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass, field
12
+ from typing import Protocol, runtime_checkable
13
+
14
+
15
+ @dataclass
16
+ class Usage:
17
+ """Running tally of tokens spent, so callers can report real cost."""
18
+
19
+ prompt_tokens: int = 0
20
+ completion_tokens: int = 0
21
+
22
+ @property
23
+ def total(self) -> int:
24
+ return self.prompt_tokens + self.completion_tokens
25
+
26
+ def add(self, prompt: int, completion: int) -> None:
27
+ self.prompt_tokens += prompt
28
+ self.completion_tokens += completion
29
+
30
+
31
+ @runtime_checkable
32
+ class LLMClient(Protocol):
33
+ """Minimal contract any provider adapter must satisfy.
34
+
35
+ Kept tiny on purpose: one text-in/text-out call. Concrete adapters
36
+ (OpenAI, local models, etc.) live in submodules and record usage.
37
+ """
38
+
39
+ usage: Usage
40
+
41
+ def complete(self, system: str, user: str) -> str:
42
+ """Return the model's text response to a system+user prompt."""
43
+ ...
44
+
45
+
46
+ @dataclass
47
+ class EchoClient:
48
+ """A no-network stub used in tests and offline development.
49
+
50
+ It never contacts a provider; it just returns a fixed string. This
51
+ lets the whole heal/extract pipeline run in CI without API keys.
52
+ """
53
+
54
+ reply: str = "{}"
55
+ usage: Usage = field(default_factory=Usage)
56
+
57
+ def complete(self, system: str, user: str) -> str:
58
+ # Pretend we spent a few tokens so cost tracking has something to show.
59
+ self.usage.add(prompt=len(user) // 4, completion=len(self.reply) // 4)
60
+ return self.reply
@@ -0,0 +1,6 @@
1
+ """Concrete LLM provider adapters.
2
+
3
+ Each adapter implements the :class:`scrapemend.llm.LLMClient` protocol
4
+ (a ``usage`` ledger plus a ``complete(system, user)`` call) and is kept
5
+ import-light so the core library never hard-depends on any one provider.
6
+ """
@@ -0,0 +1,83 @@
1
+ """DeepSeek-backed adapter for selector healing.
2
+
3
+ DeepSeek exposes an OpenAI-compatible API, so this adapter reuses the
4
+ ``openai`` SDK and only swaps the base URL, key source, and model name.
5
+ It supports JSON output mode, which is exactly what the healer wants
6
+ (a strict JSON object mapping fields to selectors).
7
+
8
+ DeepSeek is a strong default here: it is reachable from mainland China
9
+ without extra setup and is cheap enough that infrequent healing calls
10
+ are effectively free. Install the SDK with:
11
+
12
+ pip install "scrapemend[openai]"
13
+
14
+ and set DEEPSEEK_API_KEY in your environment.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import os
20
+
21
+ from scrapemend.llm import Usage
22
+
23
+ #: DeepSeek's OpenAI-compatible endpoint.
24
+ DEEPSEEK_BASE_URL = "https://api.deepseek.com"
25
+
26
+ #: Alias that always points at DeepSeek's current main chat model, so this
27
+ #: adapter keeps working across model version bumps.
28
+ DEEPSEEK_DEFAULT_MODEL = "deepseek-chat"
29
+
30
+
31
+ class DeepSeekClient:
32
+ """Calls DeepSeek's chat API (OpenAI-compatible) and tracks usage.
33
+
34
+ Args:
35
+ model: Chat model name. Defaults to the ``deepseek-chat`` alias.
36
+ api_key: Overrides the ``DEEPSEEK_API_KEY`` environment variable.
37
+ base_url: Overrides the default DeepSeek endpoint (useful for
38
+ proxies or OpenAI-compatible gateways).
39
+ temperature: Kept low for deterministic, parseable selector JSON.
40
+ """
41
+
42
+ def __init__(
43
+ self,
44
+ model: str = DEEPSEEK_DEFAULT_MODEL,
45
+ api_key: str | None = None,
46
+ base_url: str = DEEPSEEK_BASE_URL,
47
+ temperature: float = 0.0,
48
+ ) -> None:
49
+ try:
50
+ from openai import OpenAI
51
+ except ImportError as exc: # pragma: no cover - import guard
52
+ raise ImportError(
53
+ "The DeepSeek adapter uses the OpenAI SDK. "
54
+ 'Install it with: pip install "scrapemend[openai]"'
55
+ ) from exc
56
+
57
+ key = api_key or os.environ.get("DEEPSEEK_API_KEY")
58
+ if not key:
59
+ raise ValueError(
60
+ "No API key. Pass api_key=... or set DEEPSEEK_API_KEY in the environment."
61
+ )
62
+
63
+ self._client = OpenAI(api_key=key, base_url=base_url)
64
+ self.model = model
65
+ self.temperature = temperature
66
+ self.usage = Usage()
67
+
68
+ def complete(self, system: str, user: str) -> str:
69
+ resp = self._client.chat.completions.create(
70
+ model=self.model,
71
+ temperature=self.temperature,
72
+ response_format={"type": "json_object"},
73
+ messages=[
74
+ {"role": "system", "content": system},
75
+ {"role": "user", "content": user},
76
+ ],
77
+ )
78
+ if resp.usage is not None:
79
+ self.usage.add(
80
+ prompt=resp.usage.prompt_tokens,
81
+ completion=resp.usage.completion_tokens,
82
+ )
83
+ return resp.choices[0].message.content or "{}"
@@ -0,0 +1,67 @@
1
+ """OpenAI-backed adapter for selector healing.
2
+
3
+ Implements the LLMClient protocol. The ``openai`` package is imported
4
+ lazily so that merely importing scrapemend never requires it — only
5
+ constructing an :class:`OpenAIClient` does. Install with:
6
+
7
+ pip install "scrapemend[openai]"
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import os
13
+
14
+ from scrapemend.llm import Usage
15
+
16
+
17
+ class OpenAIClient:
18
+ """Calls an OpenAI chat model and tracks token usage.
19
+
20
+ Args:
21
+ model: Chat model name. A small, cheap model is the sensible
22
+ default since healing is infrequent and the task is narrow.
23
+ api_key: Overrides the ``OPENAI_API_KEY`` environment variable.
24
+ temperature: Kept low for deterministic, parseable selector JSON.
25
+ """
26
+
27
+ def __init__(
28
+ self,
29
+ model: str = "gpt-4o-mini",
30
+ api_key: str | None = None,
31
+ temperature: float = 0.0,
32
+ ) -> None:
33
+ try:
34
+ from openai import OpenAI
35
+ except ImportError as exc: # pragma: no cover - import guard
36
+ raise ImportError(
37
+ "The OpenAI adapter needs the 'openai' package. "
38
+ 'Install it with: pip install "scrapemend[openai]"'
39
+ ) from exc
40
+
41
+ key = api_key or os.environ.get("OPENAI_API_KEY")
42
+ if not key:
43
+ raise ValueError(
44
+ "No API key. Pass api_key=... or set OPENAI_API_KEY in the environment."
45
+ )
46
+
47
+ self._client = OpenAI(api_key=key)
48
+ self.model = model
49
+ self.temperature = temperature
50
+ self.usage = Usage()
51
+
52
+ def complete(self, system: str, user: str) -> str:
53
+ resp = self._client.chat.completions.create(
54
+ model=self.model,
55
+ temperature=self.temperature,
56
+ response_format={"type": "json_object"},
57
+ messages=[
58
+ {"role": "system", "content": system},
59
+ {"role": "user", "content": user},
60
+ ],
61
+ )
62
+ if resp.usage is not None:
63
+ self.usage.add(
64
+ prompt=resp.usage.prompt_tokens,
65
+ completion=resp.usage.completion_tokens,
66
+ )
67
+ return resp.choices[0].message.content or "{}"
scrapemend/result.py ADDED
@@ -0,0 +1,37 @@
1
+ """Return type carrying extracted data plus a little provenance.
2
+
3
+ Beyond the data itself, each extraction reports *how* it was produced:
4
+ whether cached selectors worked or the LLM had to heal them, and the
5
+ token cost of any healing. That provenance is what later powers cost
6
+ tracking — the shared layer the planned LLM-tooling product reuses.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass, field
12
+ from typing import Any
13
+
14
+
15
+ @dataclass
16
+ class ExtractionResult:
17
+ """Outcome of a single extraction.
18
+
19
+ Attributes:
20
+ data: Mapping of field name to extracted value(s).
21
+ healed: True when selectors had to be repaired by the LLM this run.
22
+ from_cache: True when every field was served from cached selectors.
23
+ tokens: Tokens spent on healing this run (0 when nothing healed).
24
+ notes: Human-readable diagnostics, e.g. which fields failed.
25
+ """
26
+
27
+ data: dict[str, Any] = field(default_factory=dict)
28
+ healed: bool = False
29
+ from_cache: bool = True
30
+ tokens: int = 0
31
+ notes: list[str] = field(default_factory=list)
32
+
33
+ def __getitem__(self, key: str) -> Any:
34
+ return self.data[key]
35
+
36
+ def get(self, key: str, default: Any = None) -> Any:
37
+ return self.data.get(key, default)
scrapemend/schema.py ADDED
@@ -0,0 +1,51 @@
1
+ """User-facing description of *what* to extract.
2
+
3
+ A ``Schema`` is a plain, declarative description of the fields the user
4
+ wants off a page. It carries no selectors — selectors are discovered and
5
+ healed at runtime and live in the cache, not here. This keeps the thing
6
+ the user writes stable even as target sites change their markup.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass, field
12
+
13
+
14
+ @dataclass
15
+ class Field:
16
+ """One piece of data to pull off a page.
17
+
18
+ Args:
19
+ name: Key the extracted value appears under in the result.
20
+ describe: Plain-language description used by the LLM to locate the
21
+ value when selectors need to be (re)discovered. Example:
22
+ "the product price including currency symbol".
23
+ many: If True, collect every match as a list instead of the first.
24
+ attr: Optional HTML attribute to read (e.g. "href"). When None, the
25
+ element's text content is used.
26
+ """
27
+
28
+ name: str
29
+ describe: str
30
+ many: bool = False
31
+ attr: str | None = None
32
+
33
+
34
+ @dataclass
35
+ class Schema:
36
+ """A named collection of :class:`Field` objects.
37
+
38
+ The ``name`` is used as the cache namespace so that selectors healed
39
+ for one schema/site pair do not collide with another.
40
+ """
41
+
42
+ name: str
43
+ fields: list[Field] = field(default_factory=list)
44
+
45
+ def add(self, name: str, describe: str, *, many: bool = False, attr: str | None = None) -> Schema:
46
+ """Add a field and return self so calls can be chained."""
47
+ self.fields.append(Field(name=name, describe=describe, many=many, attr=attr))
48
+ return self
49
+
50
+ def field_names(self) -> list[str]:
51
+ return [f.name for f in self.fields]
scrapemend/scraper.py ADDED
@@ -0,0 +1,99 @@
1
+ """The public entry point tying cache, parsing, and healing together.
2
+
3
+ Flow per extraction:
4
+ 1. Try cached selectors (free, fast).
5
+ 2. If any required field fails to match, the page likely changed —
6
+ invoke the LLM healer, re-extract, and cache the repaired selectors.
7
+
8
+ This "cache first, heal only on miss" policy is the whole point: you get
9
+ AI robustness without paying an LLM call on every request.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import httpx
15
+ from selectolax.lexbor import LexborHTMLParser as HTMLParser
16
+
17
+ from scrapemend.cache import SelectorCache
18
+ from scrapemend.healer import heal
19
+ from scrapemend.llm import EchoClient, LLMClient
20
+ from scrapemend.result import ExtractionResult
21
+ from scrapemend.schema import Schema
22
+
23
+
24
+ class Scraper:
25
+ """Extracts data described by a :class:`Schema` from web pages.
26
+
27
+ Args:
28
+ client: LLM adapter used for healing. Defaults to an offline stub
29
+ so the library imports and runs without any API key; pass a
30
+ real provider adapter to enable actual healing.
31
+ cache: Selector cache. Defaults to a JSON file in the cwd.
32
+ """
33
+
34
+ def __init__(self, client: LLMClient | None = None, cache: SelectorCache | None = None) -> None:
35
+ self.client = client or EchoClient()
36
+ self.cache = cache or SelectorCache()
37
+
38
+ def fetch(self, url: str) -> str:
39
+ resp = httpx.get(url, follow_redirects=True, timeout=30)
40
+ resp.raise_for_status()
41
+ return resp.text
42
+
43
+ def extract(self, url: str, schema: Schema, *, html: str | None = None) -> ExtractionResult:
44
+ """Extract ``schema`` from ``url`` (or supplied ``html``)."""
45
+ page = html if html is not None else self.fetch(url)
46
+ tree = HTMLParser(page)
47
+
48
+ selectors = self.cache.get(schema.name, url)
49
+ if selectors is not None:
50
+ result = self._apply(tree, schema, selectors)
51
+ if not result.notes: # every field matched
52
+ result.from_cache = True
53
+ return result
54
+ # Cached selectors went stale — fall through to healing.
55
+
56
+ return self._heal_and_extract(url, schema, page, tree)
57
+
58
+ def _heal_and_extract(
59
+ self, url: str, schema: Schema, page: str, tree: HTMLParser
60
+ ) -> ExtractionResult:
61
+ before = self.client.usage.total
62
+ selectors = heal(schema, page, self.client)
63
+ spent = self.client.usage.total - before
64
+
65
+ if not selectors:
66
+ return ExtractionResult(
67
+ from_cache=False,
68
+ healed=True,
69
+ tokens=spent,
70
+ notes=["healing failed: model returned no usable selectors"],
71
+ )
72
+
73
+ result = self._apply(tree, schema, selectors)
74
+ result.from_cache = False
75
+ result.healed = True
76
+ result.tokens = spent
77
+ if not any("no match" in n for n in result.notes):
78
+ self.cache.put(schema.name, url, selectors)
79
+ return result
80
+
81
+ @staticmethod
82
+ def _apply(tree: HTMLParser, schema: Schema, selectors: dict[str, str]) -> ExtractionResult:
83
+ out = ExtractionResult()
84
+ for f in schema.fields:
85
+ sel = selectors.get(f.name)
86
+ nodes = tree.css(sel) if sel else []
87
+ if not nodes:
88
+ out.notes.append(f"{f.name}: no match")
89
+ out.data[f.name] = [] if f.many else None
90
+ continue
91
+ values = [_read(n, f.attr) for n in nodes]
92
+ out.data[f.name] = values if f.many else values[0]
93
+ return out
94
+
95
+
96
+ def _read(node, attr: str | None) -> str:
97
+ if attr is not None:
98
+ return node.attributes.get(attr, "") or ""
99
+ return node.text(strip=True)
@@ -0,0 +1,118 @@
1
+ Metadata-Version: 2.5
2
+ Name: scrapemend
3
+ Version: 0.0.1
4
+ Summary: Self-healing web scraping for Python. Describe what you want in plain language; when a site changes and your selectors break, an LLM repairs them automatically.
5
+ Project-URL: Homepage, https://github.com/FengYue427/scrapemend
6
+ Project-URL: Repository, https://github.com/FengYue427/scrapemend
7
+ Project-URL: Issues, https://github.com/FengYue427/scrapemend/issues
8
+ Author: FengYue427
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: ai,crawler,data-extraction,llm,scraping,self-healing,web-scraping
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Requires-Python: >=3.10
23
+ Requires-Dist: httpx>=0.27
24
+ Requires-Dist: selectolax>=0.3.21
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest>=8; extra == 'dev'
27
+ Requires-Dist: ruff>=0.6; extra == 'dev'
28
+ Provides-Extra: openai
29
+ Requires-Dist: openai>=1.40; extra == 'openai'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # scrapemend
33
+
34
+ **Self-healing web scraping for Python.** Describe the data you want once.
35
+ scrapemend extracts it with cached CSS selectors — and when a site changes
36
+ its markup and those selectors break, a language model inspects the new page,
37
+ repairs the selectors, and caches the fix. The next run is free again.
38
+
39
+ > Traditional scrapers break every time a site is redesigned. scrapemend
40
+ > treats a broken selector as a cache miss, not a crash.
41
+
42
+ ---
43
+
44
+ ## Why
45
+
46
+ Every scraper hits the same wall: you write selectors, the site ships a
47
+ redesign, your selectors silently return nothing, and you go rewrite them by
48
+ hand. Cloud "AI scraping" services solve the robustness problem but are
49
+ expensive, send your target data through someone else's servers, and run an
50
+ LLM call on *every* request.
51
+
52
+ scrapemend takes the middle path:
53
+
54
+ - **Cache first.** Known selectors run with zero LLM cost — it's just parsing.
55
+ - **Heal on miss.** The LLM is only called when selectors stop matching.
56
+ - **Self-hosted.** It's a library, not a service. Your keys, your data.
57
+
58
+ ## Install
59
+
60
+ ```bash
61
+ pip install scrapemend
62
+ ```
63
+
64
+ ## Quickstart
65
+
66
+ ```python
67
+ from scrapemend import Scraper, Schema
68
+
69
+ schema = (
70
+ Schema("product")
71
+ .add("title", "the product name")
72
+ .add("price", "the product price with currency")
73
+ )
74
+
75
+ scraper = Scraper() # offline stub by default; plug in a real LLM to heal
76
+ result = scraper.extract("https://shop.example/p/1", schema)
77
+
78
+ print(result["title"], result["price"])
79
+ print("healed:", result.healed, "tokens:", result.tokens)
80
+ ```
81
+
82
+ You describe *what* you want in plain language. You never hand-write a CSS
83
+ selector — scrapemend discovers them, and re-discovers them when they break.
84
+
85
+ ## How it works
86
+
87
+ ```
88
+ extract(url, schema)
89
+ │
90
+ ├─ cached selectors exist? ──► apply them
91
+ │ ├─ all fields matched ──► return (free)
92
+ │ └─ some field missed ──┐
93
+ │ ▼
94
+ └─ no cache ─────────────────────────────────► LLM heals selectors
95
+ apply, cache, return
96
+ ```
97
+
98
+ Because healing only runs on a miss, steady-state scraping costs nothing in
99
+ tokens. You pay the model exactly when a site actually changes.
100
+
101
+ ## Status
102
+
103
+ Early alpha (`0.0.1`). The core pipeline — cache, parse, heal, re-cache — works
104
+ and is covered by tests that run fully offline. A real OpenAI adapter is the
105
+ next step; the LLM layer is provider-agnostic by design.
106
+
107
+ ## Roadmap
108
+
109
+ - [x] Cache-first / heal-on-miss pipeline
110
+ - [x] Offline, key-free test path
111
+ - [ ] OpenAI adapter
112
+ - [ ] Local-model adapter
113
+ - [x] HTML condensing for cheaper prompts
114
+ - [ ] Pluggable cost/budget guards
115
+
116
+ ## License
117
+
118
+ MIT
@@ -0,0 +1,15 @@
1
+ scrapemend/__init__.py,sha256=UiSV6ygPHOosILNHykrgudKqNSOru3JkiF_cGAWXRh0,543
2
+ scrapemend/cache.py,sha256=yoNEupOBNc1rRoqtlWyGal7LQZmv8lAxK3dBTw3y7ME,2066
3
+ scrapemend/condense.py,sha256=vSgPpwQWvLchxYgVTC8WDh-RRg71Jej3tXdP0oKKYk8,2047
4
+ scrapemend/healer.py,sha256=yWNWwpRN7Eh3_tGRVyC12SFVdrKZ-j4jopqRdfKZ57Y,1639
5
+ scrapemend/llm.py,sha256=QekhWfk9Syih6BPynKqwrNecbMq4CNSCPL6BtdJkbtI,1807
6
+ scrapemend/result.py,sha256=W4OWeLr0-tfE_0Y3mvidZWC9sHImyDUP8cs93l5Sycg,1261
7
+ scrapemend/schema.py,sha256=w1wsSG5C32-nlbuhTXS1GsjY-nIXcedKvK4gnUUfX6E,1707
8
+ scrapemend/scraper.py,sha256=vhc2iR7VRSGhpaaSH8vKFOvo3ndzvbJZ-f_IzWRBPJs,3667
9
+ scrapemend/providers/__init__.py,sha256=QUGFwbWQ6v83e-lhYtjFL5xZReLXIQgER7gZAZAjF8M,256
10
+ scrapemend/providers/deepseek_client.py,sha256=aiOTJk2MyQBnZz_pKcFD262h_5xYlN8MnRhhfPKHyXk,2896
11
+ scrapemend/providers/openai_client.py,sha256=EqYdx7FxXjNaj991qVyfiZblkD-ADEbrZx-Oo3a0vmw,2195
12
+ scrapemend-0.0.1.dist-info/METADATA,sha256=XbKxRvfEyjQE-vCzBULmlpeaXVfPwWaaZR9g9EJg-D4,4227
13
+ scrapemend-0.0.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
14
+ scrapemend-0.0.1.dist-info/licenses/LICENSE,sha256=IPczPPdAVaxv7gISsUvTNmtnHRpQoOo119XeNIZXSnw,1067
15
+ scrapemend-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.4
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 OriginTown
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.