jevrake 0.1.0__py3-none-any.whl

jevrake/__init__.py

@@ -0,0 +1,13 @@
+"""jevrake: selector-free web scraping with TypeSafe AI's Jev."""
+
+from ._internal.api import rake
+from ._internal.models import ExtractedValue, RakeItem, RakeResult
+from ._internal.version import VERSION as __version__
+
+__all__ = [
+    "ExtractedValue",
+    "RakeItem",
+    "RakeResult",
+    "__version__",
+    "rake",
+]

jevrake/_internal/__init__.py

@@ -0,0 +1 @@
+"""Internal implementation. Not part of the public API; may change at any time."""

jevrake/_internal/api.py

@@ -0,0 +1,138 @@
+"""Extract records from each page with ``rake()``.
+
+For each URL: download the page, split it into items (one for a single-record page), collect
+candidate values, ask Jev to choose one per field, and validate each item with the user's model.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from pydantic import BaseModel, ValidationError
+
+from . import collect, dom, jev
+from .client import MODEL, JevClient, TypeSafeJevClient
+from .exceptions import FetchError, JevAuthError, JevError
+from .fetch import Fetcher, HttpxFetcher
+from .models import ExtractionTarget, ItemBlock, JevResponse, RakeItem, RakeResult
+from .schema import build_targets_from_model
+
+DEFAULT_MIN_CONFIDENCE = 0.9
+MAX_ITEMS_PER_REQUEST = 20
+
+RequestHook = Callable[[str, dict[str, Any]], None]
+
+
+def _build_failed_result[T: BaseModel](model: type[T], url: str, error: str) -> RakeResult[T]:
+    return RakeResult[model](url=url, items=[], request_ids=[], errors=[error])  # type: ignore[valid-type]
+
+
+def _format_validation_errors(error: ValidationError) -> list[str]:
+    return [f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" for err in error.errors()]
+
+
+def _rake_page[T: BaseModel](
+    url: str,
+    model: type[T],
+    targets: list[ExtractionTarget],
+    fetcher: Fetcher,
+    client: JevClient | None,
+    min_confidence: float,
+    on_request: RequestHook | None,
+) -> RakeResult[T]:
+    try:
+        html = fetcher.fetch(url)
+    except FetchError as e:
+        return _build_failed_result(model, url, str(e))
+
+    page = dom.parse_page(html)
+    blocks = collect.find_item_blocks(page, targets)
+    numbered: list[tuple[int | None, ItemBlock]] = (
+        [(None, blocks[0])] if blocks[0].is_whole_page else list(enumerate(blocks))
+    )
+
+    items: list[RakeItem[T]] = []
+    request_ids: list[str] = []
+    errors: list[str] = []
+    for start in range(0, len(numbered), MAX_ITEMS_PER_REQUEST):
+        chunk = numbered[start : start + MAX_ITEMS_PER_REQUEST]
+        candidates_by_item = {item: block.candidates for item, block in chunk}
+        request = jev.build_jev_request(url, page, targets, candidates_by_item)
+        if on_request is not None:
+            on_request(url, jev.build_jev_payload(request, MODEL))
+        if client is None:
+            continue
+
+        response = JevResponse()
+        if request.questions:
+            try:
+                response = client.ask(request.state, request.questions)
+            except JevAuthError:
+                raise
+            except JevError as e:
+                first, last = chunk[0][0], chunk[-1][0]
+                errors.append(str(e) if first is None else f"items {first}-{last}: {e}")
+                continue
+            if response.request_id:
+                request_ids.append(response.request_id)
+
+        for item, _ in chunk:
+            prefix = "" if item is None else f"item {item}: "
+            outcome = jev.parse_jev_response(request, response, targets, min_confidence, item)
+            if outcome.errors:
+                errors.extend(prefix + e for e in outcome.errors)
+                continue
+            try:
+                data = model.model_validate(outcome.values)
+            except ValidationError as e:
+                errors.extend(prefix + m for m in _format_validation_errors(e))
+                continue
+            items.append(RakeItem[model](data=data, fields=outcome.fields))  # type: ignore[valid-type]
+
+    return RakeResult[model](url=url, items=items, request_ids=request_ids, errors=errors)  # type: ignore[valid-type]
+
+
+def _rake_page_safely[T: BaseModel](url: str, model: type[T], *args: Any) -> RakeResult[T]:
+    try:
+        return _rake_page(url, model, *args)
+    except JevAuthError:
+        raise
+    except Exception as e:  # one bad page must not stop the others
+        return _build_failed_result(model, url, f"{type(e).__name__}: {e}")
+
+
+def rake[T: BaseModel](
+    urls: Sequence[str],
+    model: type[T],
+    *,
+    min_confidence: float = DEFAULT_MIN_CONFIDENCE,
+    fetcher: Fetcher | None = None,
+    client: JevClient | None = None,
+    dry_run: bool = False,
+    on_request: RequestHook | None = None,
+) -> list[RakeResult[T]]:
+    """Extract records of ``model`` from each URL, one page at a time, in the order of ``urls``.
+
+    Raises :class:`JevAuthError` if the API key is missing or rejected.
+    """
+    targets = build_targets_from_model(model)
+
+    created: list[HttpxFetcher | TypeSafeJevClient] = []
+    if fetcher is None:
+        fetcher = HttpxFetcher()
+        created.append(fetcher)
+    if dry_run:
+        client = None
+    elif client is None:
+        client = TypeSafeJevClient()
+        created.append(client)
+
+    try:
+        return [
+            _rake_page_safely(url, model, targets, fetcher, client, min_confidence, on_request)
+            for url in urls
+        ]
+    finally:
+        for resource in created:
+            resource.close()

jevrake/_internal/client.py

@@ -0,0 +1,81 @@
+"""Send the question to Jev (TypeSafe AI API) and return its answer.
+
+Wraps the official SDK so the rest of jevrake does not depend on SDK types or exceptions.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Protocol
+
+import typesafe_sdk as ts
+
+from .exceptions import JevAuthError, JevError, MissingAPIKeyError
+from .models import JevChoiceAnswer, JevResponse
+
+if TYPE_CHECKING:
+    import httpx2
+
+API_KEY_ENV = "TYPESAFE_API_KEY"
+MODEL = "jev-latest"
+TIMEOUT = 30.0
+MAX_RETRIES = 3
+
+MISSING_API_KEY_MESSAGE = f"""\
+{API_KEY_ENV} is not set.
+Create an API key at https://console.typesafe.ai and set it with one of:
+  export {API_KEY_ENV}=<your API key>
+  cp .envrc.example .envrc  # fill in the key, then run `direnv allow`"""
+
+
+class JevClient(Protocol):
+    def ask(self, state: Mapping[str, Any], questions: Mapping[str, Any]) -> JevResponse: ...
+
+
+class TypeSafeJevClient:
+    """:class:`JevClient` backed by ``typesafe_sdk.TypeSafeClient``.
+
+    429 / 5xx responses are retried with exponential backoff (up to ``MAX_RETRIES`` times) by
+    the SDK's retry policy.
+    """
+
+    def __init__(self, *, transport: httpx2.BaseTransport | None = None) -> None:
+        api_key = os.environ.get(API_KEY_ENV)
+
+        if not api_key:
+            raise MissingAPIKeyError(MISSING_API_KEY_MESSAGE)
+
+        self._client = ts.TypeSafeClient(
+            api_key=api_key,
+            retry=ts.RetryPolicy(max_retries=MAX_RETRIES, timeout=None),
+            timeout=TIMEOUT,
+            transport=transport,
+        )
+
+    def ask(self, state: Mapping[str, Any], questions: Mapping[str, Any]) -> JevResponse:
+        try:
+            response = self._client.system_one(state=dict(state), questions=questions, model=MODEL)
+        except (ts.TypeSafeAuthenticationError, ts.TypeSafePermissionDeniedError) as e:
+            raise JevAuthError(
+                f"Jev API rejected the API key (HTTP {e.status}). Check {API_KEY_ENV}."
+            ) from None
+        except ts.TypeSafeAPITimeoutError:
+            raise JevError("Jev API request timed out") from None
+        except ts.TypeSafeAPIError as e:
+            raise JevError(f"Jev API error (HTTP {e.status}): {e.body}") from None
+        except ts.TypeSafeError as e:
+            raise JevError(f"Jev API error: {e}") from None
+
+        return JevResponse(
+            choices={
+                name: JevChoiceAnswer(
+                    choice=a.choice, confidence=a.confidence, probabilities=dict(a.probabilities)
+                )
+                for name, a in response.choices.items()
+            },
+            request_id=response.request_id,
+        )
+
+    def close(self) -> None:
+        self._client.close()

jevrake/_internal/collect.py

@@ -0,0 +1,210 @@
+"""Collect possible values (candidates) for each target from the page.
+
+A list page is split into item blocks (one per record); any other page is one block. Text targets
+take titles and text elements; number targets take prices and other numbers. Candidates are
+de-duplicated and, if there are too many, only the highest-scoring ones are kept.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from collections.abc import Iterator
+from typing import Any
+
+from .convert import NUMBER_RE, convert_text, normalize_text
+from .exceptions import CoercionError
+from .models import ExtractionCandidate, ExtractionTarget, ItemBlock, ParsedPage, TextElement
+
+MAX_CANDIDATES = 254  # plus "__none__" = 255 choices
+MAX_TEXT_CHARS = 200
+_PREFIX_LABEL_CHARS = 20
+MIN_LIST_ITEMS = 3
+MIN_LIST_TEXT_SHARE = 0.5
+MAX_SHORT_TEXT_CHARS = 40
+
+
+def _format_tag(element: TextElement) -> str:
+    attrs = "".join(
+        f" {key}={element.attrs[key]}"
+        for key in ("class", "id", "itemprop")
+        if element.attrs.get(key)
+    )
+    return f"<{element.tag}{attrs}>"
+
+
+def _build_context(element: TextElement, label: str | None = None) -> dict[str, str]:
+    context = {"tag": _format_tag(element)}
+    label = label or element.label
+    if label and label != element.text:
+        context["label"] = label
+    if element.heading_level:
+        context["heading"] = f"h{element.heading_level}"
+    return context
+
+
+def _find_prefix_label(text: str, start: int) -> str | None:
+    prefix = text[:start].strip().rstrip(":：").strip()
+    if 0 < len(prefix) <= _PREFIX_LABEL_CHARS and not re.search(r"\d", prefix[-3:]):
+        return prefix
+    return None
+
+
+def _find_text_candidates(page: ParsedPage, block: ItemBlock) -> Iterator[ExtractionCandidate]:
+    if block.is_whole_page and page.title:
+        yield ExtractionCandidate(
+            text=page.title,
+            normalized_value=None,
+            context={"tag": "<title>", "source": "page title"},
+        )
+    if block.is_whole_page and page.og_title:
+        yield ExtractionCandidate(
+            text=page.og_title,
+            normalized_value=None,
+            context={"tag": "<meta og:title>", "source": "og:title"},
+        )
+    for element in block.elements:
+        if 2 <= len(element.text) <= MAX_TEXT_CHARS:
+            yield ExtractionCandidate(
+                text=element.text,
+                normalized_value=None,
+                context=_build_context(element),
+            )
+
+
+def _find_number_candidates(block: ItemBlock) -> Iterator[ExtractionCandidate]:
+    for element in block.elements:
+        text = normalize_text(element.text)
+        for m in NUMBER_RE.finditer(text):
+            context = _build_context(element, _find_prefix_label(text, m.start()))
+            if m.group("prefix") or m.group("suffix"):
+                context["unit"] = (m.group("prefix") or m.group("suffix")).strip()
+            yield ExtractionCandidate(
+                text=m.group(0).strip(),
+                normalized_value=None,
+                context=context,
+            )
+
+
+def _split_terms(text: str) -> set[str]:
+    """Matching terms: whole words for Latin text, character bigrams for CJK text."""
+    s = normalize_text(text).lower()
+    terms = {w for w in re.findall(r"[a-z]{3,}", s)}
+    for run in re.findall(r"[^\x00-\x7f\s\W]+", s):
+        terms |= {run[i : i + 2] for i in range(len(run) - 1)}
+    return terms
+
+
+def _calculate_score(
+    candidate: ExtractionCandidate, target: ExtractionTarget, frequency: int
+) -> float:
+    ctx = candidate.context
+    score = 0.0
+    name_terms = _split_terms(target.name.replace("_", " "))
+    wanted = _split_terms(target.description) | name_terms
+    label = ctx.get("label")
+    if label:
+        score += 1.0
+        if _split_terms(label) & wanted:
+            score += 2.0
+    if _split_terms(ctx.get("tag", "").replace("-", " ").replace("_", " ")) & name_terms:
+        score += 1.5
+    heading = ctx.get("heading")
+    if heading and target.kind == "str":
+        score += {"h1": 3.0, "h2": 2.0}.get(heading, 1.0)
+    if ctx.get("source"):
+        score += 3.0
+    score += 0.5 * min(frequency - 1, 3)
+    if target.kind != "str" and ctx.get("unit"):
+        score += 1.0
+    if target.kind == "str" and len(candidate.text) > 80:
+        score -= 1.0
+    return score
+
+
+def remove_duplicates(candidates: list[ExtractionCandidate]) -> list[ExtractionCandidate]:
+    """Keep one candidate per ``normalized_value``: the one with the most context."""
+    best: dict[Any, int] = {}
+    for i, candidate in enumerate(candidates):
+        key = candidate.normalized_value
+        if key not in best or len(candidate.context) > len(candidates[best[key]].context):
+            best[key] = i
+    return [candidates[i] for i in sorted(best.values())]
+
+
+def keep_top_candidates(candidates: list[ExtractionCandidate]) -> list[ExtractionCandidate]:
+    """Keep the ``MAX_CANDIDATES`` highest-scoring candidates, preserving document order."""
+    if len(candidates) <= MAX_CANDIDATES:
+        return candidates
+    ranked = sorted(range(len(candidates)), key=lambda i: -candidates[i].score)
+    keep = sorted(ranked[:MAX_CANDIDATES])
+    return [candidates[i] for i in keep]
+
+
+def generate_candidates(
+    page: ParsedPage, block: ItemBlock, target: ExtractionTarget
+) -> list[ExtractionCandidate]:
+    converted: list[ExtractionCandidate] = []
+    raw_candidates = (
+        _find_text_candidates(page, block)
+        if target.kind == "str"
+        else _find_number_candidates(block)
+    )
+    for raw in raw_candidates:
+        try:
+            value = convert_text(raw.text, target.kind)
+        except CoercionError:
+            continue
+        converted.append(raw.model_copy(update={"normalized_value": value}))
+
+    frequency = Counter(c.normalized_value for c in converted)
+    unique = remove_duplicates(converted)
+    scored = [
+        c.model_copy(update={"score": _calculate_score(c, target, frequency[c.normalized_value])})
+        for c in unique
+    ]
+    return keep_top_candidates(scored)
+
+
+def _count_text(elements: list[TextElement]) -> int:
+    return sum(len(e.text) for e in elements)
+
+
+def _build_block(
+    page: ParsedPage,
+    elements: list[TextElement],
+    is_whole_page: bool,
+    targets: list[ExtractionTarget],
+) -> ItemBlock:
+    """Make a block and collect the candidates of every target in it."""
+    block = ItemBlock(elements=elements, is_whole_page=is_whole_page)
+    candidates = {t.name: generate_candidates(page, block, t) for t in targets}
+    return block.model_copy(update={"candidates": candidates})
+
+
+def find_item_blocks(page: ParsedPage, targets: list[ExtractionTarget]) -> list[ItemBlock]:
+    """Return one block per item on a list page, or the whole page as a single block.
+
+    Each block comes with the candidates of every target, so they are collected only once.
+
+    Every block of the chosen group is returned, so an item that misses a value is reported later
+    instead of silently disappearing.
+    """
+    required = [t for t in targets if not t.is_optional]
+    page_text = _count_text(page.elements)
+    best: list[ItemBlock] = []
+    best_items = 0
+    for group in page.repeated_groups:
+        blocks = [_build_block(page, elements, False, targets) for elements in group]
+        items = sum(
+            1
+            for block in blocks
+            if len(block.elements) >= 2
+            and any(len(e.text) <= MAX_SHORT_TEXT_CHARS for e in block.elements)
+            and all(block.candidates[t.name] for t in required)
+        )
+        text_share = _count_text([e for b in blocks for e in b.elements]) / max(page_text, 1)
+        is_list = items >= MIN_LIST_ITEMS and text_share >= MIN_LIST_TEXT_SHARE
+        if is_list and items > best_items:
+            best, best_items = blocks, items
+    return best or [_build_block(page, page.elements, True, targets)]

jevrake/_internal/convert.py

@@ -0,0 +1,78 @@
+"""Convert candidate text into the target's type, e.g. ``"¥3,980"`` -> ``3980``.
+
+Used to turn page text into values and to drop text that cannot be converted.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections.abc import Callable
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+from .exceptions import CoercionError
+
+
+def normalize_text(text: str) -> str:
+    """NFKC-normalize (full-width -> half-width) and collapse whitespace."""
+    return " ".join(unicodedata.normalize("NFKC", text).split())
+
+
+CURRENCY_PREFIXES = "¥$€£"
+CURRENCY_SUFFIXES = ("円", "ドル", "ユーロ", "USD", "JPY", "EUR", "yen")
+
+NUMBER_RE = re.compile(
+    r"(?<![\w.,/:\-])"
+    r"(?P<prefix>[¥$€£]\s?)?"
+    r"(?P<number>-?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?)"
+    r"(?P<suffix>\s?(?:円|ドル|ユーロ|USD|JPY|EUR|yen))?"
+    r"(?!\d|,\d|[.:/\-]\d|[年月日時分秒])"
+)
+
+
+_PLAIN_NUMBER_RE = re.compile(r"-?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?")
+
+
+def parse_number(text: str) -> Decimal:
+    s = normalize_text(text).replace("\u2212", "-")
+    s = s.strip(CURRENCY_PREFIXES + " ")
+    for suffix in CURRENCY_SUFFIXES:
+        s = s.removesuffix(suffix).strip()
+    if not _PLAIN_NUMBER_RE.fullmatch(s):
+        raise CoercionError(f"not a number: {text!r}")
+    try:
+        return Decimal(s.replace(",", ""))
+    except InvalidOperation:
+        raise CoercionError(f"not a number: {text!r}") from None
+
+
+def _convert_to_int(text: str) -> int:
+    value = parse_number(text)
+    if value != value.to_integral_value():
+        raise CoercionError(f"not an integer: {text!r}")
+    return int(value)
+
+
+def _convert_to_str(text: str) -> str:
+    s = " ".join(text.split())
+    if not s:
+        raise CoercionError("empty string")
+    return s
+
+
+_CONVERTERS: dict[str, Callable[[str], Any]] = {
+    "str": _convert_to_str,
+    "int": _convert_to_int,
+    "float": lambda t: float(parse_number(t)),
+    "decimal": parse_number,
+}
+
+
+def convert_text(text: str, kind: str) -> Any:
+    """Convert ``text`` to the Python value for ``kind``. Raises :class:`CoercionError`."""
+    try:
+        converter = _CONVERTERS[kind]
+    except KeyError:
+        raise CoercionError(f"unsupported type: {kind}") from None
+    return converter(text)