jevrake 0.1.0__py3-none-any.whl

jevrake/_internal/dom.py

@@ -0,0 +1,200 @@
+"""Turn raw HTML into a ``ParsedPage``.
+
+Removes scripts, navigation, footers and hidden elements, then collects the page title, the main
+text, and each element that holds text together with its nearby label. It also finds blocks that
+repeat (such as the items of a list page).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TypeGuard
+
+from selectolax.parser import HTMLParser, Node
+
+from .models import ParsedPage, TextElement
+
+MAX_MAIN_TEXT_CHARS = 8000
+
+REMOVED_TAGS = ("script", "style", "nav", "footer", "noscript", "template", "svg", "iframe")
+_HIDDEN_STYLE_RE = re.compile(r"display\s*:\s*none|visibility\s*:\s*hidden", re.IGNORECASE)
+_BLOCK_TAGS = {
+    "p", "div", "li", "tr", "td", "th", "dt", "dd", "h1", "h2", "h3", "h4", "h5", "h6",
+    "section", "article", "main", "header", "aside", "table", "ul", "ol", "dl", "br",
+    "blockquote", "pre", "figure", "figcaption", "form", "label",
+}  # fmt: skip
+MAX_LABEL_CHARS = 30
+MIN_REPEATED_BLOCKS = 3
+
+
+def _is_element(node: Node | None) -> TypeGuard[Node]:
+    return node is not None and not node.tag.startswith(("-", "_"))
+
+
+def _is_hidden(node: Node) -> bool:
+    attrs = node.attributes
+    if "hidden" in attrs or attrs.get("aria-hidden") == "true":
+        return True
+    return bool(_HIDDEN_STYLE_RE.search(attrs.get("style") or ""))
+
+
+def clean_html(html: str) -> HTMLParser:
+    """Parse ``html`` and remove non-content elements and hidden elements."""
+    tree = HTMLParser(html)
+    for tag in REMOVED_TAGS:
+        for node in tree.css(tag):
+            node.decompose()
+    if tree.body is not None:
+        hidden = [n for n in tree.body.traverse() if _is_element(n) and _is_hidden(n)]
+        # Nodes inside a hidden ancestor are removed together with it.
+        for node in hidden:
+            if not any(_is_hidden(a) for a in _get_ancestors(node)):
+                node.decompose()
+    return tree
+
+
+def _get_ancestors(node: Node) -> list[Node]:
+    result: list[Node] = []
+    current = node.parent
+    while _is_element(current):
+        result.append(current)
+        current = current.parent
+    return result
+
+
+def _get_own_text(node: Node) -> str:
+    parts = [child.text() for child in node.iter(include_text=True) if child.tag == "-text"]
+    return " ".join(" ".join(parts).split())
+
+
+def _get_full_text(node: Node) -> str:
+    return " ".join(node.text(deep=True, separator=" ").split())
+
+
+def _find_previous_element(node: Node) -> Node | None:
+    sibling = node.prev
+    while sibling is not None and not _is_element(sibling):
+        sibling = sibling.prev
+    return sibling
+
+
+def _keep_if_short(text: str) -> str | None:
+    text = text.strip().rstrip(":：").strip()
+    return text if 0 < len(text) <= MAX_LABEL_CHARS else None
+
+
+def _find_label(node: Node) -> str | None:
+    """Find a nearby label: table header, definition term, or a short preceding element.
+
+    Looks at the node itself and its parent only; going further up tends to pick unrelated text.
+    """
+    current: Node | None = node
+    for _ in range(2):
+        if not _is_element(current):
+            return None
+        if current.tag in ("td", "th", "dd"):
+            prev = _find_previous_element(current)
+            if prev is not None and prev.tag in ("th", "td", "dt"):
+                return _keep_if_short(_get_full_text(prev))
+        prev = _find_previous_element(current)
+        if prev is not None:
+            label = _keep_if_short(_get_full_text(prev))
+            if label:
+                return label
+        for attr in ("aria-label", "itemprop"):
+            if current.attributes.get(attr):
+                return _keep_if_short(current.attributes[attr] or "")
+        current = current.parent
+    return None
+
+
+def _find_heading_level(node: Node) -> int | None:
+    current: Node | None = node
+    for _ in range(3):
+        if not _is_element(current):
+            return None
+        if re.fullmatch(r"h[1-6]", current.tag):
+            return int(current.tag[1])
+        current = current.parent
+    return None
+
+
+def _build_main_text(body: Node) -> str:
+    lines: list[str] = []
+    buffer: list[str] = []
+
+    def add_line() -> None:
+        line = " ".join(" ".join(buffer).split())
+        if line:
+            lines.append(line)
+        buffer.clear()
+
+    for node in body.traverse(include_text=True):
+        if node.tag == "-text":
+            buffer.append(node.text())
+        elif node.tag in _BLOCK_TAGS:
+            add_line()
+    add_line()
+    return "\n".join(lines)[:MAX_MAIN_TEXT_CHARS]
+
+
+def _find_repeated_blocks(body: Node) -> list[list[Node]]:
+    """Find children of the same parent that share a tag and class, at least 3 of them."""
+    groups: list[list[Node]] = []
+    for parent in body.traverse():
+        if not _is_element(parent):
+            continue
+        by_shape: dict[tuple[str, str], list[Node]] = {}
+        for child in parent.iter():
+            if _is_element(child) and _get_full_text(child):
+                shape = (child.tag, child.attributes.get("class") or "")
+                by_shape.setdefault(shape, []).append(child)
+        groups.extend(b for b in by_shape.values() if len(b) >= MIN_REPEATED_BLOCKS)
+    return groups
+
+
+def parse_page(html: str) -> ParsedPage:
+    raw = HTMLParser(html)
+    title_node = raw.css_first("title")
+    title = " ".join(title_node.text().split()) if title_node else None
+    og_node = raw.css_first('meta[property="og:title"]')
+    og_title = (og_node.attributes.get("content") or "").strip() or None if og_node else None
+
+    tree = clean_html(html)
+    body = tree.body
+    if body is None:
+        return ParsedPage(title=title or None, og_title=og_title, main_text="", elements=[])
+
+    block_nodes = _find_repeated_blocks(body)
+    block_index = {
+        node.mem_id: (g, b) for g, group in enumerate(block_nodes) for b, node in enumerate(group)
+    }
+    repeated_groups: list[list[list[TextElement]]] = [[[] for _ in g] for g in block_nodes]
+
+    elements: list[TextElement] = []
+    for node in body.traverse():
+        if not _is_element(node) or node.tag == "body":
+            continue
+        text = _get_own_text(node)
+        if not text:
+            continue
+        element = TextElement(
+            text=text,
+            tag=node.tag,
+            attrs={k: v or "" for k, v in node.attributes.items()},
+            label=_find_label(node),
+            heading_level=_find_heading_level(node),
+        )
+        elements.append(element)
+        for container in (node, *_get_ancestors(node)):
+            if container.mem_id in block_index:
+                g, b = block_index[container.mem_id]
+                repeated_groups[g][b].append(element)
+
+    return ParsedPage(
+        title=title or None,
+        og_title=og_title,
+        main_text=_build_main_text(body),
+        elements=elements,
+        repeated_groups=repeated_groups,
+    )

jevrake/_internal/exceptions.py

@@ -0,0 +1,25 @@
+"""Exceptions raised by jevrake."""
+
+
+class SchemaError(ValueError):
+    """A field definition is invalid."""
+
+
+class CoercionError(ValueError):
+    """Candidate text cannot be converted to the target's type."""
+
+
+class FetchError(Exception):
+    """A page could not be fetched."""
+
+
+class JevError(Exception):
+    """A Jev request failed."""
+
+
+class JevAuthError(JevError):
+    """Authentication failed (missing key, 401, 403). The whole run should stop."""
+
+
+class MissingAPIKeyError(JevAuthError):
+    """The API key environment variable is not set."""

jevrake/_internal/fetch.py

@@ -0,0 +1,75 @@
+"""Download the HTML of each page.
+
+Waits ``delay`` seconds between requests to the same site so the site is not overloaded.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Protocol
+from urllib.parse import urlsplit
+
+import httpx
+
+from .exceptions import FetchError
+from .version import USER_AGENT
+
+DEFAULT_DELAY = 1.0
+TIMEOUT = 30.0
+
+
+class Fetcher(Protocol):
+    def fetch(self, url: str) -> str:
+        """Return the HTML of ``url``. Raises :class:`FetchError`."""
+
+
+def _get_origin(url: str) -> str:
+    parts = urlsplit(url)
+    return f"{parts.scheme}://{parts.netloc}"
+
+
+class HttpxFetcher:
+    def __init__(
+        self,
+        *,
+        delay: float = DEFAULT_DELAY,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self.delay = delay
+        self._client = httpx.Client(
+            headers={"User-Agent": USER_AGENT},
+            timeout=TIMEOUT,
+            follow_redirects=True,
+            transport=transport,
+        )
+        self._last_access: dict[str, float] = {}
+
+    def _get_with_delay(self, url: str) -> httpx.Response:
+        """GET ``url`` after waiting so requests to the same host are ``delay`` seconds apart."""
+        origin = _get_origin(url)
+        last = self._last_access.get(origin)
+        if last is not None:
+            wait = last + self.delay - time.monotonic()
+            if wait > 0:
+                time.sleep(wait)
+        self._last_access[origin] = time.monotonic()
+        return self._client.get(url)
+
+    def fetch(self, url: str) -> str:
+        if urlsplit(url).scheme not in ("http", "https"):
+            raise FetchError(f"not an http/https URL: {url}")
+        try:
+            response = self._get_with_delay(url)
+        except httpx.TimeoutException:
+            raise FetchError(f"request timed out: {url}") from None
+        except httpx.HTTPError as e:
+            raise FetchError(f"request failed: {url}: {e}") from None
+        if not response.is_success:
+            raise FetchError(f"HTTP {response.status_code}: {url}")
+        content_type = response.headers.get("content-type", "")
+        if content_type and not any(t in content_type for t in ("html", "xml", "text/plain")):
+            raise FetchError(f"not an HTML page ({content_type}): {url}")
+        return response.text
+
+    def close(self) -> None:
+        self._client.close()

jevrake/_internal/jev.py

@@ -0,0 +1,181 @@
+"""Build the question for Jev, and read Jev's answer.
+
+Has no network access; sending the question is done by client.py.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from typesafe_sdk import Choice
+
+from .models import (
+    ExtractedValue,
+    ExtractionCandidate,
+    ExtractionOutcome,
+    ExtractionTarget,
+    JevRequest,
+    JevResponse,
+    ParsedPage,
+)
+
+NONE_CHOICE = "__none__"
+NONE_DESCRIPTION = "None of these is the requested value."
+MAX_CHOICES = 255
+MAX_DESCRIPTION_TEXT = 200
+
+CHOICE_INSTRUCTIONS = (
+    'Which option is "{description}"? Choose the option that represents the {name} of this page.'
+)
+
+ITEM_CHOICE_INSTRUCTIONS = (
+    'Which option is "{description}"? All options come from one item of a list on this page; '
+    "choose the option that represents the {name} of that item."
+)
+
+
+def build_question_key(name: str, item: int | None) -> str:
+    """``price`` for a single-record page, ``item3.price`` for item 3 of a list page."""
+    return name if item is None else f"item{item}.{name}"
+
+
+def describe_candidate(candidate: ExtractionCandidate) -> str:
+    """e.g. ``"¥3,980" | <span class=price> | nearby label: 販売価格``"""
+    text = candidate.text
+    if len(text) > MAX_DESCRIPTION_TEXT:
+        text = text[:MAX_DESCRIPTION_TEXT] + "…"
+    parts = [json.dumps(text, ensure_ascii=False)]
+    ctx = candidate.context
+    if ctx.get("source"):
+        parts.append(ctx["source"])
+    elif ctx.get("tag"):
+        parts.append(ctx["tag"])
+    if ctx.get("heading"):
+        parts.append(f"heading {ctx['heading']}")
+    if ctx.get("label"):
+        parts.append(f"nearby label: {ctx['label']}")
+    return " | ".join(parts)
+
+
+def build_jev_state(url: str, page: ParsedPage) -> dict[str, str]:
+    return {
+        "url": url,
+        "page_title": page.title or page.og_title or "",
+        "main_text": page.main_text,
+    }
+
+
+def build_jev_request(
+    url: str,
+    page: ParsedPage,
+    targets: list[ExtractionTarget],
+    candidates_by_item: dict[int | None, dict[str, list[ExtractionCandidate]]],
+) -> JevRequest:
+    """Build one request with a question per item and target.
+
+    ``candidates_by_item`` maps an item number (``None`` for a single-record page) to the
+    candidates of each target in that item.
+    """
+    request = JevRequest(state=build_jev_state(url, page))
+    for item, candidates in candidates_by_item.items():
+        for target in targets:
+            key = build_question_key(target.name, item)
+            chosen = candidates.get(target.name, [])[: MAX_CHOICES - 1]
+            if not chosen:
+                request.skipped.append(key)
+                continue
+            names = [f"c{i}" for i in range(len(chosen))]
+            criteria = {n: describe_candidate(c) for n, c in zip(names, chosen, strict=True)}
+            criteria[NONE_CHOICE] = NONE_DESCRIPTION
+            template = CHOICE_INSTRUCTIONS if item is None else ITEM_CHOICE_INSTRUCTIONS
+            request.questions[key] = Choice(
+                instructions=template.format(
+                    description=target.description, name=target.name, item=item
+                ),
+                criteria=criteria,
+            )
+            request.options[key] = dict(zip(names, chosen, strict=True))
+    return request
+
+
+def build_jev_payload(request: JevRequest, model: str) -> dict[str, Any]:
+    """The request body as it would be sent (for ``--dry-run``)."""
+    return {
+        "model": model,
+        "state": request.state,
+        "questions": {
+            name: q.model_dump(mode="json", exclude_none=True)
+            for name, q in request.questions.items()
+        },
+    }
+
+
+def _record_missing_value(
+    target: ExtractionTarget, values: dict[str, Any], errors: list[str], reason: str
+) -> None:
+    if not target.is_optional:
+        errors.append(f"{target.name}: {reason}")
+    elif target.required:
+        values[target.name] = None
+
+
+def parse_jev_response(
+    request: JevRequest,
+    response: JevResponse,
+    targets: list[ExtractionTarget],
+    min_confidence: float,
+    item: int | None = None,
+) -> ExtractionOutcome:
+    """Turn Jev's answers for one item (``None`` for a single-record page) into values."""
+    fields: dict[str, ExtractedValue] = {}
+    values: dict[str, Any] = {}
+    errors: list[str] = []
+
+    for target in targets:
+        name = target.name
+        key = build_question_key(name, item)
+        if key in request.skipped:
+            fields[name] = ExtractedValue(value=None, confidence=None, probability=0.0, source=None)
+            _record_missing_value(target, values, errors, "no candidates found on the page")
+            continue
+
+        answer = response.choices.get(key)
+        if answer is None:
+            errors.append(f"{name}: missing from the Jev response")
+            continue
+        probability = answer.probabilities.get(answer.choice, answer.confidence)
+        if answer.choice == NONE_CHOICE:
+            fields[name] = ExtractedValue(
+                value=None, confidence=answer.confidence, probability=probability, source=None
+            )
+            _record_missing_value(target, values, errors, "no matching value (Jev chose __none__)")
+            continue
+        candidate = request.options[key].get(answer.choice)
+        if candidate is None:
+            errors.append(f"{name}: Jev returned an unknown choice: {answer.choice!r}")
+            continue
+        if answer.confidence < min_confidence:
+            fields[name] = ExtractedValue(
+                value=None,
+                confidence=answer.confidence,
+                probability=probability,
+                source=candidate.text,
+            )
+            _record_missing_value(
+                target,
+                values,
+                errors,
+                f"confidence {answer.confidence:.2f} is below {min_confidence:.2f} "
+                f"(rejected {candidate.text!r})",
+            )
+            continue
+        fields[name] = ExtractedValue(
+            value=candidate.normalized_value,
+            confidence=answer.confidence,
+            probability=probability,
+            source=candidate.text,
+        )
+        values[name] = candidate.normalized_value
+
+    return ExtractionOutcome(fields=fields, values=values, errors=errors)

jevrake/_internal/models.py

@@ -0,0 +1,131 @@
+"""Data passed between the steps of ``rake()``, and the results returned to users."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+from typesafe_sdk import Choice
+
+ExtractionKind = Literal["str", "int", "float", "decimal"]
+
+
+class ExtractionTarget(BaseModel):
+    """Internal description of one field to extract."""
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    description: str
+    kind: ExtractionKind
+    nullable: bool = False
+    required: bool = True
+
+    @property
+    def is_optional(self) -> bool:
+        return self.nullable or not self.required
+
+
+class TargetDefinition(BaseModel):
+    """One entry of the CLI ``--fields`` JSON."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    description: str | None = None
+    type: ExtractionKind = "str"
+
+
+class TextElement(BaseModel):
+    """An element that directly contains text, with information about its surroundings."""
+
+    model_config = ConfigDict(frozen=True)
+
+    text: str
+    tag: str
+    attrs: dict[str, str]
+    label: str | None = None
+    heading_level: int | None = None
+
+
+class ParsedPage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    title: str | None
+    og_title: str | None
+    main_text: str
+    elements: list[TextElement]
+    repeated_groups: list[list[list[TextElement]]] = []
+
+
+class ExtractionCandidate(BaseModel):
+    """A value found on the page that may be the answer for a field."""
+
+    model_config = ConfigDict(frozen=True)
+
+    text: str
+    normalized_value: Any
+    context: dict[str, str] = {}
+    score: float = 0.0
+
+
+class ItemBlock(BaseModel):
+    """The part of a page that holds one record: the whole page, or one item of a list page."""
+
+    model_config = ConfigDict(frozen=True)
+
+    elements: list[TextElement]
+    is_whole_page: bool
+    candidates: dict[str, list[ExtractionCandidate]] = {}
+
+
+class JevRequest(BaseModel):
+    state: dict[str, str]
+    questions: dict[str, Choice] = {}
+    options: dict[str, dict[str, ExtractionCandidate]] = {}
+    skipped: list[str] = []
+
+
+class JevChoiceAnswer(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    choice: str
+    confidence: float
+    probabilities: dict[str, float]
+
+
+class JevResponse(BaseModel):
+    """The parts of a Jev response jevrake uses, independent of the SDK types."""
+
+    model_config = ConfigDict(frozen=True)
+
+    choices: dict[str, JevChoiceAnswer] = {}
+    request_id: str | None = None
+
+
+class ExtractedValue(BaseModel):
+    value: Any | None
+    confidence: float | None
+    probability: float
+    source: str | None
+
+
+class ExtractionOutcome(BaseModel):
+    fields: dict[str, ExtractedValue]
+    values: dict[str, Any]
+    errors: list[str]
+
+
+class RakeItem[T: BaseModel](BaseModel):
+    """One record found on a page."""
+
+    data: T
+    fields: dict[str, ExtractedValue]
+
+
+class RakeResult[T: BaseModel](BaseModel):
+    """All records found on one page."""
+
+    url: str
+    items: list[RakeItem[T]]
+    request_ids: list[str]
+    errors: list[str]