scrapemend 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapemend/__init__.py +14 -0
- scrapemend/cache.py +57 -0
- scrapemend/condense.py +60 -0
- scrapemend/healer.py +49 -0
- scrapemend/llm.py +60 -0
- scrapemend/providers/__init__.py +6 -0
- scrapemend/providers/deepseek_client.py +83 -0
- scrapemend/providers/openai_client.py +67 -0
- scrapemend/result.py +37 -0
- scrapemend/schema.py +51 -0
- scrapemend/scraper.py +99 -0
- scrapemend-0.0.1.dist-info/METADATA +118 -0
- scrapemend-0.0.1.dist-info/RECORD +15 -0
- scrapemend-0.0.1.dist-info/WHEEL +4 -0
- scrapemend-0.0.1.dist-info/licenses/LICENSE +21 -0
scrapemend/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""scrapemend — self-healing web scraping for Python.
|
|
2
|
+
|
|
3
|
+
Describe the data you want once. scrapemend extracts it with cached
|
|
4
|
+
selectors; when a site changes its markup and those selectors stop
|
|
5
|
+
matching, a language model inspects the new page and repairs them
|
|
6
|
+
automatically, then caches the fix so the next run is free again.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from scrapemend.result import ExtractionResult
|
|
10
|
+
from scrapemend.schema import Field, Schema
|
|
11
|
+
from scrapemend.scraper import Scraper
|
|
12
|
+
|
|
13
|
+
__all__ = ["ExtractionResult", "Field", "Schema", "Scraper"]
|
|
14
|
+
__version__ = "0.0.1"
|
scrapemend/cache.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""On-disk selector cache keyed by (schema, site).
|
|
2
|
+
|
|
3
|
+
The cache is what makes the common path free: once selectors are known
|
|
4
|
+
for a given schema and host, extraction is pure parsing with no LLM call.
|
|
5
|
+
Healing only writes here when the cached selectors stop matching.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from urllib.parse import urlparse
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SelectorCache:
|
|
16
|
+
"""Maps a schema+host to the CSS selectors last known to work.
|
|
17
|
+
|
|
18
|
+
Stored as a single JSON file so it is trivial to inspect, diff, and
|
|
19
|
+
commit to a repo if a team wants to share healed selectors. (Shared,
|
|
20
|
+
hosted cache sync is a candidate Pro feature later.)
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, path: str | Path = ".scrapemend-cache.json") -> None:
|
|
24
|
+
self.path = Path(path)
|
|
25
|
+
self._data: dict[str, dict[str, str]] = {}
|
|
26
|
+
self._load()
|
|
27
|
+
|
|
28
|
+
def _load(self) -> None:
|
|
29
|
+
if self.path.exists():
|
|
30
|
+
try:
|
|
31
|
+
self._data = json.loads(self.path.read_text(encoding="utf-8"))
|
|
32
|
+
except (json.JSONDecodeError, OSError):
|
|
33
|
+
self._data = {}
|
|
34
|
+
|
|
35
|
+
def _flush(self) -> None:
|
|
36
|
+
self.path.write_text(
|
|
37
|
+
json.dumps(self._data, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def _key(schema_name: str, url: str) -> str:
|
|
42
|
+
host = urlparse(url).netloc or "unknown"
|
|
43
|
+
return f"{schema_name}@{host}"
|
|
44
|
+
|
|
45
|
+
def get(self, schema_name: str, url: str) -> dict[str, str] | None:
|
|
46
|
+
"""Return cached {field_name: css_selector} or None if unknown."""
|
|
47
|
+
return self._data.get(self._key(schema_name, url))
|
|
48
|
+
|
|
49
|
+
def put(self, schema_name: str, url: str, selectors: dict[str, str]) -> None:
|
|
50
|
+
"""Store healed selectors for this schema+host and persist."""
|
|
51
|
+
self._data[self._key(schema_name, url)] = selectors
|
|
52
|
+
self._flush()
|
|
53
|
+
|
|
54
|
+
def drop(self, schema_name: str, url: str) -> None:
|
|
55
|
+
"""Forget selectors for a schema+host (forces a re-heal next run)."""
|
|
56
|
+
self._data.pop(self._key(schema_name, url), None)
|
|
57
|
+
self._flush()
|
scrapemend/condense.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Shrink HTML before it goes into a healing prompt.
|
|
2
|
+
|
|
3
|
+
The model only needs the page's *structure* to pick CSS selectors, not
|
|
4
|
+
its scripts, styles, or inline data payloads. Stripping those cuts prompt
|
|
5
|
+
tokens sharply (cheaper healing) and removes noise that can mislead the
|
|
6
|
+
model (more accurate selectors). This directly serves the library's
|
|
7
|
+
"robust but cheap" promise.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
from selectolax.lexbor import LexborHTMLParser
|
|
15
|
+
|
|
16
|
+
#: Tags that carry no structural signal for selector discovery.
|
|
17
|
+
_DROP_TAGS = ("script", "style", "noscript", "template", "svg", "iframe")
|
|
18
|
+
|
|
19
|
+
#: The only attributes that matter for building CSS selectors.
|
|
20
|
+
_KEEP_ATTRS = ("class", "id")
|
|
21
|
+
|
|
22
|
+
_WS = re.compile(r"\s+")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def condense(html: str, limit: int = 6000) -> str:
|
|
26
|
+
"""Return a stripped-down version of ``html`` for prompting.
|
|
27
|
+
|
|
28
|
+
Removes non-structural tags, drops every attribute except class/id,
|
|
29
|
+
collapses whitespace, and finally caps length as a hard token guard.
|
|
30
|
+
|
|
31
|
+
Falls back to a plain truncation if parsing fails for any reason, so
|
|
32
|
+
healing never breaks just because a page is malformed.
|
|
33
|
+
"""
|
|
34
|
+
try:
|
|
35
|
+
tree = LexborHTMLParser(html)
|
|
36
|
+
except Exception: # noqa: BLE001 # pragma: no cover - last-resort guard; never let healing crash on a weird page
|
|
37
|
+
return _WS.sub(" ", html)[:limit]
|
|
38
|
+
|
|
39
|
+
for tag in _DROP_TAGS:
|
|
40
|
+
for node in tree.css(tag):
|
|
41
|
+
node.decompose()
|
|
42
|
+
|
|
43
|
+
body = tree.body if tree.body is not None else tree.root
|
|
44
|
+
if body is None:
|
|
45
|
+
return _WS.sub(" ", html)[:limit]
|
|
46
|
+
|
|
47
|
+
for node in body.traverse(include_text=False):
|
|
48
|
+
attrs = getattr(node, "attributes", None)
|
|
49
|
+
if not attrs:
|
|
50
|
+
continue
|
|
51
|
+
for name in list(attrs.keys()):
|
|
52
|
+
if name not in _KEEP_ATTRS:
|
|
53
|
+
try:
|
|
54
|
+
del node.attrs[name]
|
|
55
|
+
except (KeyError, TypeError): # pragma: no cover - defensive
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
cleaned = body.html or ""
|
|
59
|
+
cleaned = _WS.sub(" ", cleaned).strip()
|
|
60
|
+
return cleaned[:limit]
|
scrapemend/healer.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Selector discovery/repair via the LLM.
|
|
2
|
+
|
|
3
|
+
Given a page and a schema, ask the model for one CSS selector per field.
|
|
4
|
+
The healer is only invoked when cached selectors are missing or have
|
|
5
|
+
stopped matching, so its token cost is paid rarely, not per request.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
|
|
12
|
+
from scrapemend.condense import condense
|
|
13
|
+
from scrapemend.llm import LLMClient
|
|
14
|
+
from scrapemend.schema import Schema
|
|
15
|
+
|
|
16
|
+
_SYSTEM = (
|
|
17
|
+
"You map data fields to CSS selectors for a web page. "
|
|
18
|
+
"Return ONLY a JSON object mapping each field name to a single CSS "
|
|
19
|
+
"selector string that selects the element holding that field's value. "
|
|
20
|
+
"No prose, no code fences."
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def build_prompt(schema: Schema, html: str) -> str:
|
|
25
|
+
lines = [f"- {f.name}: {f.describe}" for f in schema.fields]
|
|
26
|
+
return (
|
|
27
|
+
"Fields to locate:\n"
|
|
28
|
+
+ "\n".join(lines)
|
|
29
|
+
+ "\n\nPage HTML (condensed, possibly truncated):\n"
|
|
30
|
+
+ condense(html)
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def heal(schema: Schema, html: str, client: LLMClient) -> dict[str, str]:
|
|
35
|
+
"""Ask the LLM for fresh selectors. Returns {field_name: selector}.
|
|
36
|
+
|
|
37
|
+
Unknown or malformed replies yield an empty mapping rather than
|
|
38
|
+
raising, so the caller can record a clean failure and move on.
|
|
39
|
+
"""
|
|
40
|
+
reply = client.complete(_SYSTEM, build_prompt(schema, html))
|
|
41
|
+
try:
|
|
42
|
+
parsed = json.loads(reply)
|
|
43
|
+
except json.JSONDecodeError:
|
|
44
|
+
return {}
|
|
45
|
+
if not isinstance(parsed, dict):
|
|
46
|
+
return {}
|
|
47
|
+
# Keep only known fields with string selectors.
|
|
48
|
+
valid = set(schema.field_names())
|
|
49
|
+
return {k: v for k, v in parsed.items() if k in valid and isinstance(v, str)}
|
scrapemend/llm.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""The shared LLM layer.
|
|
2
|
+
|
|
3
|
+
This module is deliberately generic: a provider-agnostic client plus a
|
|
4
|
+
token/cost ledger. scrapemend uses it only for selector healing, but it
|
|
5
|
+
carries no scraping concepts, so the planned LLM-developer-tooling product
|
|
6
|
+
can reuse it wholesale. Keep it free of any scraping imports.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Protocol, runtime_checkable
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class Usage:
|
|
17
|
+
"""Running tally of tokens spent, so callers can report real cost."""
|
|
18
|
+
|
|
19
|
+
prompt_tokens: int = 0
|
|
20
|
+
completion_tokens: int = 0
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def total(self) -> int:
|
|
24
|
+
return self.prompt_tokens + self.completion_tokens
|
|
25
|
+
|
|
26
|
+
def add(self, prompt: int, completion: int) -> None:
|
|
27
|
+
self.prompt_tokens += prompt
|
|
28
|
+
self.completion_tokens += completion
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@runtime_checkable
|
|
32
|
+
class LLMClient(Protocol):
|
|
33
|
+
"""Minimal contract any provider adapter must satisfy.
|
|
34
|
+
|
|
35
|
+
Kept tiny on purpose: one text-in/text-out call. Concrete adapters
|
|
36
|
+
(OpenAI, local models, etc.) live in submodules and record usage.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
usage: Usage
|
|
40
|
+
|
|
41
|
+
def complete(self, system: str, user: str) -> str:
|
|
42
|
+
"""Return the model's text response to a system+user prompt."""
|
|
43
|
+
...
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class EchoClient:
|
|
48
|
+
"""A no-network stub used in tests and offline development.
|
|
49
|
+
|
|
50
|
+
It never contacts a provider; it just returns a fixed string. This
|
|
51
|
+
lets the whole heal/extract pipeline run in CI without API keys.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
reply: str = "{}"
|
|
55
|
+
usage: Usage = field(default_factory=Usage)
|
|
56
|
+
|
|
57
|
+
def complete(self, system: str, user: str) -> str:
|
|
58
|
+
# Pretend we spent a few tokens so cost tracking has something to show.
|
|
59
|
+
self.usage.add(prompt=len(user) // 4, completion=len(self.reply) // 4)
|
|
60
|
+
return self.reply
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""DeepSeek-backed adapter for selector healing.
|
|
2
|
+
|
|
3
|
+
DeepSeek exposes an OpenAI-compatible API, so this adapter reuses the
|
|
4
|
+
``openai`` SDK and only swaps the base URL, key source, and model name.
|
|
5
|
+
It supports JSON output mode, which is exactly what the healer wants
|
|
6
|
+
(a strict JSON object mapping fields to selectors).
|
|
7
|
+
|
|
8
|
+
DeepSeek is a strong default here: it is reachable from mainland China
|
|
9
|
+
without extra setup and is cheap enough that infrequent healing calls
|
|
10
|
+
are effectively free. Install the SDK with:
|
|
11
|
+
|
|
12
|
+
pip install "scrapemend[openai]"
|
|
13
|
+
|
|
14
|
+
and set DEEPSEEK_API_KEY in your environment.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
|
|
21
|
+
from scrapemend.llm import Usage
|
|
22
|
+
|
|
23
|
+
#: DeepSeek's OpenAI-compatible endpoint.
|
|
24
|
+
DEEPSEEK_BASE_URL = "https://api.deepseek.com"
|
|
25
|
+
|
|
26
|
+
#: Alias that always points at DeepSeek's current main chat model, so this
|
|
27
|
+
#: adapter keeps working across model version bumps.
|
|
28
|
+
DEEPSEEK_DEFAULT_MODEL = "deepseek-chat"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DeepSeekClient:
|
|
32
|
+
"""Calls DeepSeek's chat API (OpenAI-compatible) and tracks usage.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
model: Chat model name. Defaults to the ``deepseek-chat`` alias.
|
|
36
|
+
api_key: Overrides the ``DEEPSEEK_API_KEY`` environment variable.
|
|
37
|
+
base_url: Overrides the default DeepSeek endpoint (useful for
|
|
38
|
+
proxies or OpenAI-compatible gateways).
|
|
39
|
+
temperature: Kept low for deterministic, parseable selector JSON.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
model: str = DEEPSEEK_DEFAULT_MODEL,
|
|
45
|
+
api_key: str | None = None,
|
|
46
|
+
base_url: str = DEEPSEEK_BASE_URL,
|
|
47
|
+
temperature: float = 0.0,
|
|
48
|
+
) -> None:
|
|
49
|
+
try:
|
|
50
|
+
from openai import OpenAI
|
|
51
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
52
|
+
raise ImportError(
|
|
53
|
+
"The DeepSeek adapter uses the OpenAI SDK. "
|
|
54
|
+
'Install it with: pip install "scrapemend[openai]"'
|
|
55
|
+
) from exc
|
|
56
|
+
|
|
57
|
+
key = api_key or os.environ.get("DEEPSEEK_API_KEY")
|
|
58
|
+
if not key:
|
|
59
|
+
raise ValueError(
|
|
60
|
+
"No API key. Pass api_key=... or set DEEPSEEK_API_KEY in the environment."
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
self._client = OpenAI(api_key=key, base_url=base_url)
|
|
64
|
+
self.model = model
|
|
65
|
+
self.temperature = temperature
|
|
66
|
+
self.usage = Usage()
|
|
67
|
+
|
|
68
|
+
def complete(self, system: str, user: str) -> str:
|
|
69
|
+
resp = self._client.chat.completions.create(
|
|
70
|
+
model=self.model,
|
|
71
|
+
temperature=self.temperature,
|
|
72
|
+
response_format={"type": "json_object"},
|
|
73
|
+
messages=[
|
|
74
|
+
{"role": "system", "content": system},
|
|
75
|
+
{"role": "user", "content": user},
|
|
76
|
+
],
|
|
77
|
+
)
|
|
78
|
+
if resp.usage is not None:
|
|
79
|
+
self.usage.add(
|
|
80
|
+
prompt=resp.usage.prompt_tokens,
|
|
81
|
+
completion=resp.usage.completion_tokens,
|
|
82
|
+
)
|
|
83
|
+
return resp.choices[0].message.content or "{}"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""OpenAI-backed adapter for selector healing.
|
|
2
|
+
|
|
3
|
+
Implements the LLMClient protocol. The ``openai`` package is imported
|
|
4
|
+
lazily so that merely importing scrapemend never requires it — only
|
|
5
|
+
constructing an :class:`OpenAIClient` does. Install with:
|
|
6
|
+
|
|
7
|
+
pip install "scrapemend[openai]"
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
from scrapemend.llm import Usage
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class OpenAIClient:
|
|
18
|
+
"""Calls an OpenAI chat model and tracks token usage.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
model: Chat model name. A small, cheap model is the sensible
|
|
22
|
+
default since healing is infrequent and the task is narrow.
|
|
23
|
+
api_key: Overrides the ``OPENAI_API_KEY`` environment variable.
|
|
24
|
+
temperature: Kept low for deterministic, parseable selector JSON.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
model: str = "gpt-4o-mini",
|
|
30
|
+
api_key: str | None = None,
|
|
31
|
+
temperature: float = 0.0,
|
|
32
|
+
) -> None:
|
|
33
|
+
try:
|
|
34
|
+
from openai import OpenAI
|
|
35
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
36
|
+
raise ImportError(
|
|
37
|
+
"The OpenAI adapter needs the 'openai' package. "
|
|
38
|
+
'Install it with: pip install "scrapemend[openai]"'
|
|
39
|
+
) from exc
|
|
40
|
+
|
|
41
|
+
key = api_key or os.environ.get("OPENAI_API_KEY")
|
|
42
|
+
if not key:
|
|
43
|
+
raise ValueError(
|
|
44
|
+
"No API key. Pass api_key=... or set OPENAI_API_KEY in the environment."
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
self._client = OpenAI(api_key=key)
|
|
48
|
+
self.model = model
|
|
49
|
+
self.temperature = temperature
|
|
50
|
+
self.usage = Usage()
|
|
51
|
+
|
|
52
|
+
def complete(self, system: str, user: str) -> str:
|
|
53
|
+
resp = self._client.chat.completions.create(
|
|
54
|
+
model=self.model,
|
|
55
|
+
temperature=self.temperature,
|
|
56
|
+
response_format={"type": "json_object"},
|
|
57
|
+
messages=[
|
|
58
|
+
{"role": "system", "content": system},
|
|
59
|
+
{"role": "user", "content": user},
|
|
60
|
+
],
|
|
61
|
+
)
|
|
62
|
+
if resp.usage is not None:
|
|
63
|
+
self.usage.add(
|
|
64
|
+
prompt=resp.usage.prompt_tokens,
|
|
65
|
+
completion=resp.usage.completion_tokens,
|
|
66
|
+
)
|
|
67
|
+
return resp.choices[0].message.content or "{}"
|
scrapemend/result.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Return type carrying extracted data plus a little provenance.
|
|
2
|
+
|
|
3
|
+
Beyond the data itself, each extraction reports *how* it was produced:
|
|
4
|
+
whether cached selectors worked or the LLM had to heal them, and the
|
|
5
|
+
token cost of any healing. That provenance is what later powers cost
|
|
6
|
+
tracking — the shared layer the planned LLM-tooling product reuses.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class ExtractionResult:
|
|
17
|
+
"""Outcome of a single extraction.
|
|
18
|
+
|
|
19
|
+
Attributes:
|
|
20
|
+
data: Mapping of field name to extracted value(s).
|
|
21
|
+
healed: True when selectors had to be repaired by the LLM this run.
|
|
22
|
+
from_cache: True when every field was served from cached selectors.
|
|
23
|
+
tokens: Tokens spent on healing this run (0 when nothing healed).
|
|
24
|
+
notes: Human-readable diagnostics, e.g. which fields failed.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
data: dict[str, Any] = field(default_factory=dict)
|
|
28
|
+
healed: bool = False
|
|
29
|
+
from_cache: bool = True
|
|
30
|
+
tokens: int = 0
|
|
31
|
+
notes: list[str] = field(default_factory=list)
|
|
32
|
+
|
|
33
|
+
def __getitem__(self, key: str) -> Any:
|
|
34
|
+
return self.data[key]
|
|
35
|
+
|
|
36
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
37
|
+
return self.data.get(key, default)
|
scrapemend/schema.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""User-facing description of *what* to extract.
|
|
2
|
+
|
|
3
|
+
A ``Schema`` is a plain, declarative description of the fields the user
|
|
4
|
+
wants off a page. It carries no selectors — selectors are discovered and
|
|
5
|
+
healed at runtime and live in the cache, not here. This keeps the thing
|
|
6
|
+
the user writes stable even as target sites change their markup.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Field:
|
|
16
|
+
"""One piece of data to pull off a page.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
name: Key the extracted value appears under in the result.
|
|
20
|
+
describe: Plain-language description used by the LLM to locate the
|
|
21
|
+
value when selectors need to be (re)discovered. Example:
|
|
22
|
+
"the product price including currency symbol".
|
|
23
|
+
many: If True, collect every match as a list instead of the first.
|
|
24
|
+
attr: Optional HTML attribute to read (e.g. "href"). When None, the
|
|
25
|
+
element's text content is used.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
name: str
|
|
29
|
+
describe: str
|
|
30
|
+
many: bool = False
|
|
31
|
+
attr: str | None = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Schema:
|
|
36
|
+
"""A named collection of :class:`Field` objects.
|
|
37
|
+
|
|
38
|
+
The ``name`` is used as the cache namespace so that selectors healed
|
|
39
|
+
for one schema/site pair do not collide with another.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
fields: list[Field] = field(default_factory=list)
|
|
44
|
+
|
|
45
|
+
def add(self, name: str, describe: str, *, many: bool = False, attr: str | None = None) -> Schema:
|
|
46
|
+
"""Add a field and return self so calls can be chained."""
|
|
47
|
+
self.fields.append(Field(name=name, describe=describe, many=many, attr=attr))
|
|
48
|
+
return self
|
|
49
|
+
|
|
50
|
+
def field_names(self) -> list[str]:
|
|
51
|
+
return [f.name for f in self.fields]
|
scrapemend/scraper.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""The public entry point tying cache, parsing, and healing together.
|
|
2
|
+
|
|
3
|
+
Flow per extraction:
|
|
4
|
+
1. Try cached selectors (free, fast).
|
|
5
|
+
2. If any required field fails to match, the page likely changed —
|
|
6
|
+
invoke the LLM healer, re-extract, and cache the repaired selectors.
|
|
7
|
+
|
|
8
|
+
This "cache first, heal only on miss" policy is the whole point: you get
|
|
9
|
+
AI robustness without paying an LLM call on every request.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
from selectolax.lexbor import LexborHTMLParser as HTMLParser
|
|
16
|
+
|
|
17
|
+
from scrapemend.cache import SelectorCache
|
|
18
|
+
from scrapemend.healer import heal
|
|
19
|
+
from scrapemend.llm import EchoClient, LLMClient
|
|
20
|
+
from scrapemend.result import ExtractionResult
|
|
21
|
+
from scrapemend.schema import Schema
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Scraper:
|
|
25
|
+
"""Extracts data described by a :class:`Schema` from web pages.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
client: LLM adapter used for healing. Defaults to an offline stub
|
|
29
|
+
so the library imports and runs without any API key; pass a
|
|
30
|
+
real provider adapter to enable actual healing.
|
|
31
|
+
cache: Selector cache. Defaults to a JSON file in the cwd.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, client: LLMClient | None = None, cache: SelectorCache | None = None) -> None:
|
|
35
|
+
self.client = client or EchoClient()
|
|
36
|
+
self.cache = cache or SelectorCache()
|
|
37
|
+
|
|
38
|
+
def fetch(self, url: str) -> str:
|
|
39
|
+
resp = httpx.get(url, follow_redirects=True, timeout=30)
|
|
40
|
+
resp.raise_for_status()
|
|
41
|
+
return resp.text
|
|
42
|
+
|
|
43
|
+
def extract(self, url: str, schema: Schema, *, html: str | None = None) -> ExtractionResult:
|
|
44
|
+
"""Extract ``schema`` from ``url`` (or supplied ``html``)."""
|
|
45
|
+
page = html if html is not None else self.fetch(url)
|
|
46
|
+
tree = HTMLParser(page)
|
|
47
|
+
|
|
48
|
+
selectors = self.cache.get(schema.name, url)
|
|
49
|
+
if selectors is not None:
|
|
50
|
+
result = self._apply(tree, schema, selectors)
|
|
51
|
+
if not result.notes: # every field matched
|
|
52
|
+
result.from_cache = True
|
|
53
|
+
return result
|
|
54
|
+
# Cached selectors went stale — fall through to healing.
|
|
55
|
+
|
|
56
|
+
return self._heal_and_extract(url, schema, page, tree)
|
|
57
|
+
|
|
58
|
+
def _heal_and_extract(
|
|
59
|
+
self, url: str, schema: Schema, page: str, tree: HTMLParser
|
|
60
|
+
) -> ExtractionResult:
|
|
61
|
+
before = self.client.usage.total
|
|
62
|
+
selectors = heal(schema, page, self.client)
|
|
63
|
+
spent = self.client.usage.total - before
|
|
64
|
+
|
|
65
|
+
if not selectors:
|
|
66
|
+
return ExtractionResult(
|
|
67
|
+
from_cache=False,
|
|
68
|
+
healed=True,
|
|
69
|
+
tokens=spent,
|
|
70
|
+
notes=["healing failed: model returned no usable selectors"],
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
result = self._apply(tree, schema, selectors)
|
|
74
|
+
result.from_cache = False
|
|
75
|
+
result.healed = True
|
|
76
|
+
result.tokens = spent
|
|
77
|
+
if not any("no match" in n for n in result.notes):
|
|
78
|
+
self.cache.put(schema.name, url, selectors)
|
|
79
|
+
return result
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def _apply(tree: HTMLParser, schema: Schema, selectors: dict[str, str]) -> ExtractionResult:
|
|
83
|
+
out = ExtractionResult()
|
|
84
|
+
for f in schema.fields:
|
|
85
|
+
sel = selectors.get(f.name)
|
|
86
|
+
nodes = tree.css(sel) if sel else []
|
|
87
|
+
if not nodes:
|
|
88
|
+
out.notes.append(f"{f.name}: no match")
|
|
89
|
+
out.data[f.name] = [] if f.many else None
|
|
90
|
+
continue
|
|
91
|
+
values = [_read(n, f.attr) for n in nodes]
|
|
92
|
+
out.data[f.name] = values if f.many else values[0]
|
|
93
|
+
return out
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _read(node, attr: str | None) -> str:
|
|
97
|
+
if attr is not None:
|
|
98
|
+
return node.attributes.get(attr, "") or ""
|
|
99
|
+
return node.text(strip=True)
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: scrapemend
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Self-healing web scraping for Python. Describe what you want in plain language; when a site changes and your selectors break, an LLM repairs them automatically.
|
|
5
|
+
Project-URL: Homepage, https://github.com/FengYue427/scrapemend
|
|
6
|
+
Project-URL: Repository, https://github.com/FengYue427/scrapemend
|
|
7
|
+
Project-URL: Issues, https://github.com/FengYue427/scrapemend/issues
|
|
8
|
+
Author: FengYue427
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: ai,crawler,data-extraction,llm,scraping,self-healing,web-scraping
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Requires-Dist: selectolax>=0.3.21
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
28
|
+
Provides-Extra: openai
|
|
29
|
+
Requires-Dist: openai>=1.40; extra == 'openai'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# scrapemend
|
|
33
|
+
|
|
34
|
+
**Self-healing web scraping for Python.** Describe the data you want once.
|
|
35
|
+
scrapemend extracts it with cached CSS selectors — and when a site changes
|
|
36
|
+
its markup and those selectors break, a language model inspects the new page,
|
|
37
|
+
repairs the selectors, and caches the fix. The next run is free again.
|
|
38
|
+
|
|
39
|
+
> Traditional scrapers break every time a site is redesigned. scrapemend
|
|
40
|
+
> treats a broken selector as a cache miss, not a crash.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Why
|
|
45
|
+
|
|
46
|
+
Every scraper hits the same wall: you write selectors, the site ships a
|
|
47
|
+
redesign, your selectors silently return nothing, and you go rewrite them by
|
|
48
|
+
hand. Cloud "AI scraping" services solve the robustness problem but are
|
|
49
|
+
expensive, send your target data through someone else's servers, and run an
|
|
50
|
+
LLM call on *every* request.
|
|
51
|
+
|
|
52
|
+
scrapemend takes the middle path:
|
|
53
|
+
|
|
54
|
+
- **Cache first.** Known selectors run with zero LLM cost — it's just parsing.
|
|
55
|
+
- **Heal on miss.** The LLM is only called when selectors stop matching.
|
|
56
|
+
- **Self-hosted.** It's a library, not a service. Your keys, your data.
|
|
57
|
+
|
|
58
|
+
## Install
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install scrapemend
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Quickstart
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from scrapemend import Scraper, Schema
|
|
68
|
+
|
|
69
|
+
schema = (
|
|
70
|
+
Schema("product")
|
|
71
|
+
.add("title", "the product name")
|
|
72
|
+
.add("price", "the product price with currency")
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
scraper = Scraper() # offline stub by default; plug in a real LLM to heal
|
|
76
|
+
result = scraper.extract("https://shop.example/p/1", schema)
|
|
77
|
+
|
|
78
|
+
print(result["title"], result["price"])
|
|
79
|
+
print("healed:", result.healed, "tokens:", result.tokens)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
You describe *what* you want in plain language. You never hand-write a CSS
|
|
83
|
+
selector — scrapemend discovers them, and re-discovers them when they break.
|
|
84
|
+
|
|
85
|
+
## How it works
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
extract(url, schema)
|
|
89
|
+
│
|
|
90
|
+
├─ cached selectors exist? ──► apply them
|
|
91
|
+
│ ├─ all fields matched ──► return (free)
|
|
92
|
+
│ └─ some field missed ──┐
|
|
93
|
+
│ ▼
|
|
94
|
+
└─ no cache ─────────────────────────────────► LLM heals selectors
|
|
95
|
+
apply, cache, return
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Because healing only runs on a miss, steady-state scraping costs nothing in
|
|
99
|
+
tokens. You pay the model exactly when a site actually changes.
|
|
100
|
+
|
|
101
|
+
## Status
|
|
102
|
+
|
|
103
|
+
Early alpha (`0.0.1`). The core pipeline — cache, parse, heal, re-cache — works
|
|
104
|
+
and is covered by tests that run fully offline. A real OpenAI adapter is the
|
|
105
|
+
next step; the LLM layer is provider-agnostic by design.
|
|
106
|
+
|
|
107
|
+
## Roadmap
|
|
108
|
+
|
|
109
|
+
- [x] Cache-first / heal-on-miss pipeline
|
|
110
|
+
- [x] Offline, key-free test path
|
|
111
|
+
- [ ] OpenAI adapter
|
|
112
|
+
- [ ] Local-model adapter
|
|
113
|
+
- [x] HTML condensing for cheaper prompts
|
|
114
|
+
- [ ] Pluggable cost/budget guards
|
|
115
|
+
|
|
116
|
+
## License
|
|
117
|
+
|
|
118
|
+
MIT
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
scrapemend/__init__.py,sha256=UiSV6ygPHOosILNHykrgudKqNSOru3JkiF_cGAWXRh0,543
|
|
2
|
+
scrapemend/cache.py,sha256=yoNEupOBNc1rRoqtlWyGal7LQZmv8lAxK3dBTw3y7ME,2066
|
|
3
|
+
scrapemend/condense.py,sha256=vSgPpwQWvLchxYgVTC8WDh-RRg71Jej3tXdP0oKKYk8,2047
|
|
4
|
+
scrapemend/healer.py,sha256=yWNWwpRN7Eh3_tGRVyC12SFVdrKZ-j4jopqRdfKZ57Y,1639
|
|
5
|
+
scrapemend/llm.py,sha256=QekhWfk9Syih6BPynKqwrNecbMq4CNSCPL6BtdJkbtI,1807
|
|
6
|
+
scrapemend/result.py,sha256=W4OWeLr0-tfE_0Y3mvidZWC9sHImyDUP8cs93l5Sycg,1261
|
|
7
|
+
scrapemend/schema.py,sha256=w1wsSG5C32-nlbuhTXS1GsjY-nIXcedKvK4gnUUfX6E,1707
|
|
8
|
+
scrapemend/scraper.py,sha256=vhc2iR7VRSGhpaaSH8vKFOvo3ndzvbJZ-f_IzWRBPJs,3667
|
|
9
|
+
scrapemend/providers/__init__.py,sha256=QUGFwbWQ6v83e-lhYtjFL5xZReLXIQgER7gZAZAjF8M,256
|
|
10
|
+
scrapemend/providers/deepseek_client.py,sha256=aiOTJk2MyQBnZz_pKcFD262h_5xYlN8MnRhhfPKHyXk,2896
|
|
11
|
+
scrapemend/providers/openai_client.py,sha256=EqYdx7FxXjNaj991qVyfiZblkD-ADEbrZx-Oo3a0vmw,2195
|
|
12
|
+
scrapemend-0.0.1.dist-info/METADATA,sha256=XbKxRvfEyjQE-vCzBULmlpeaXVfPwWaaZR9g9EJg-D4,4227
|
|
13
|
+
scrapemend-0.0.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
14
|
+
scrapemend-0.0.1.dist-info/licenses/LICENSE,sha256=IPczPPdAVaxv7gISsUvTNmtnHRpQoOo119XeNIZXSnw,1067
|
|
15
|
+
scrapemend-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 OriginTown
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|