jevrake 0.1.0__tar.gz

jevrake-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 RKasai127
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

jevrake-0.1.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: jevrake
+Version: 0.1.0
+Summary: Selector-free web scraping powered by TypeSafe AI's Jev
+Keywords: scraping,web-scraping,llm,pydantic,typesafe,jev
+Author: RKasai127
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Typing :: Typed
+Requires-Dist: pydantic>=2
+Requires-Dist: httpx
+Requires-Dist: selectolax
+Requires-Dist: typesafe-sdk
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/RKasai127/jevrake
+Project-URL: Repository, https://github.com/RKasai127/jevrake
+Project-URL: Issues, https://github.com/RKasai127/jevrake/issues
+Description-Content-Type: text/markdown
+
+<p align="right">English | <a href="https://github.com/RKasai127/jevrake/blob/main/README.ja.md">日本語</a></p>
+
+# jevrake — a scraping tool powered by Jev
+
+Example on [books.toscrape.com](https://books.toscrape.com/), a site made for scraping practice:
+
+```sh
+jevrake https://books.toscrape.com/ --fields '{"title": "Book title", "price": {"description": "Price", "type": "decimal"}, "availability": "Availability"}'
+```
+
+Output (1 of the 20 books on the page):
+
+```json
+{
+  "url": "https://books.toscrape.com/",
+  "data": {
+    "title": "A Light in the ...",
+    "price": "51.77",
+    "availability": "In stock"
+  },
+  "fields": {
+    "title": {"value": "A Light in the ...", "confidence": 1.0, "probability": 1.0, "source": "A Light in the ..."},
+    "price": {"value": "51.77", "confidence": 0.98, "probability": 0.99, "source": "£51.77"},
+    "availability": {"value": "In stock", "confidence": 1.0, "probability": 1.0, "source": "In stock"}
+  }
+}
+```
+
+## Install
+
+```sh
+pipx install git+https://github.com/RKasai127/jevrake
+export TYPESAFE_API_KEY=your-api-key
+```
+
+## Usage
+
+```sh
+jevrake https://example.com/item/1 --fields '{"title": "Product name", "price": {"description": "Price", "type": "int"}}'
+```
+
+Or from Python:
+
+```python
+from pydantic import BaseModel, Field
+from jevrake import rake
+
+
+class Product(BaseModel):
+    title: str = Field(description="Product name")
+    price: int = Field(description="Price")
+
+
+results = rake(["https://example.com/item/1"], Product)
+```
+
+jevrake collects every value on the page that matches each field's type, and Jev picks the one
+that fits the description. Values always come from the page.
+
+- Works on list pages too: each repeated item (e.g. each shop in a list) becomes one record
+- Types: `str`, `int`, `float`, `decimal`
+- A value that is not found, or found with confidence below 0.9, becomes `null`
+- robots.txt is not checked. Please make sure you follow each site's rules.
+
+## License
+
+MIT

jevrake-0.1.0/README.md

@@ -0,0 +1,67 @@
+<p align="right">English | <a href="https://github.com/RKasai127/jevrake/blob/main/README.ja.md">日本語</a></p>
+
+# jevrake — a scraping tool powered by Jev
+
+Example on [books.toscrape.com](https://books.toscrape.com/), a site made for scraping practice:
+
+```sh
+jevrake https://books.toscrape.com/ --fields '{"title": "Book title", "price": {"description": "Price", "type": "decimal"}, "availability": "Availability"}'
+```
+
+Output (1 of the 20 books on the page):
+
+```json
+{
+  "url": "https://books.toscrape.com/",
+  "data": {
+    "title": "A Light in the ...",
+    "price": "51.77",
+    "availability": "In stock"
+  },
+  "fields": {
+    "title": {"value": "A Light in the ...", "confidence": 1.0, "probability": 1.0, "source": "A Light in the ..."},
+    "price": {"value": "51.77", "confidence": 0.98, "probability": 0.99, "source": "£51.77"},
+    "availability": {"value": "In stock", "confidence": 1.0, "probability": 1.0, "source": "In stock"}
+  }
+}
+```
+
+## Install
+
+```sh
+pipx install git+https://github.com/RKasai127/jevrake
+export TYPESAFE_API_KEY=your-api-key
+```
+
+## Usage
+
+```sh
+jevrake https://example.com/item/1 --fields '{"title": "Product name", "price": {"description": "Price", "type": "int"}}'
+```
+
+Or from Python:
+
+```python
+from pydantic import BaseModel, Field
+from jevrake import rake
+
+
+class Product(BaseModel):
+    title: str = Field(description="Product name")
+    price: int = Field(description="Price")
+
+
+results = rake(["https://example.com/item/1"], Product)
+```
+
+jevrake collects every value on the page that matches each field's type, and Jev picks the one
+that fits the description. Values always come from the page.
+
+- Works on list pages too: each repeated item (e.g. each shop in a list) becomes one record
+- Types: `str`, `int`, `float`, `decimal`
+- A value that is not found, or found with confidence below 0.9, becomes `null`
+- robots.txt is not checked. Please make sure you follow each site's rules.
+
+## License
+
+MIT

jevrake-0.1.0/pyproject.toml

@@ -0,0 +1,79 @@
+[project]
+name = "jevrake"
+version = "0.1.0"
+description = "Selector-free web scraping powered by TypeSafe AI's Jev"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = [
+    "scraping",
+    "web-scraping",
+    "llm",
+    "pydantic",
+    "typesafe",
+    "jev",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Typing :: Typed",
+]
+requires-python = ">=3.12"
+dependencies = [
+    "pydantic>=2",
+    "httpx",
+    "selectolax",
+    "typesafe-sdk",
+]
+
+[[project.authors]]
+name = "RKasai127"
+
+[project.urls]
+Homepage = "https://github.com/RKasai127/jevrake"
+Repository = "https://github.com/RKasai127/jevrake"
+Issues = "https://github.com/RKasai127/jevrake/issues"
+
+[project.scripts]
+jevrake = "jevrake.cli:main"
+
+[build-system]
+requires = ["uv_build"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "ruff",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "UP",
+    "B",
+    "SIM",
+    "RUF",
+    "ASYNC",
+    "PT",
+]
+allowed-confusables = [
+    "（",
+    "）",
+    "：",
+    "，",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["RUF001"]

jevrake-0.1.0/pyproject.toml.orig

@@ -0,0 +1,49 @@
+[project]
+name = "jevrake"
+version = "0.1.0"
+description = "Selector-free web scraping powered by TypeSafe AI's Jev"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "RKasai127" }]
+keywords = ["scraping", "web-scraping", "llm", "pydantic", "typesafe", "jev"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Typing :: Typed",
+]
+requires-python = ">=3.12"
+dependencies = ["pydantic>=2", "httpx", "selectolax", "typesafe-sdk"]
+
+[project.urls]
+Homepage = "https://github.com/RKasai127/jevrake"
+Repository = "https://github.com/RKasai127/jevrake"
+Issues = "https://github.com/RKasai127/jevrake/issues"
+
+[project.scripts]
+jevrake = "jevrake.cli:main"
+
+[build-system]
+requires = ["uv_build"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = ["pytest", "ruff"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "RUF", "ASYNC", "PT"]
+# Full-width punctuation is normal in Japanese messages.
+allowed-confusables = ["（", "）", "：", "，"]
+
+[tool.ruff.lint.per-file-ignores]
+# Tests use full-width digits on purpose (Japanese price / date inputs).
+"tests/**" = ["RUF001"]

jevrake-0.1.0/src/jevrake/__init__.py

@@ -0,0 +1,13 @@
+"""jevrake: selector-free web scraping with TypeSafe AI's Jev."""
+
+from ._internal.api import rake
+from ._internal.models import ExtractedValue, RakeItem, RakeResult
+from ._internal.version import VERSION as __version__
+
+__all__ = [
+    "ExtractedValue",
+    "RakeItem",
+    "RakeResult",
+    "__version__",
+    "rake",
+]

jevrake-0.1.0/src/jevrake/_internal/__init__.py

@@ -0,0 +1 @@
+"""Internal implementation. Not part of the public API; may change at any time."""

jevrake-0.1.0/src/jevrake/_internal/api.py

@@ -0,0 +1,138 @@
+"""Extract records from each page with ``rake()``.
+
+For each URL: download the page, split it into items (one for a single-record page), collect
+candidate values, ask Jev to choose one per field, and validate each item with the user's model.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from pydantic import BaseModel, ValidationError
+
+from . import collect, dom, jev
+from .client import MODEL, JevClient, TypeSafeJevClient
+from .exceptions import FetchError, JevAuthError, JevError
+from .fetch import Fetcher, HttpxFetcher
+from .models import ExtractionTarget, ItemBlock, JevResponse, RakeItem, RakeResult
+from .schema import build_targets_from_model
+
+DEFAULT_MIN_CONFIDENCE = 0.9
+MAX_ITEMS_PER_REQUEST = 20
+
+RequestHook = Callable[[str, dict[str, Any]], None]
+
+
+def _build_failed_result[T: BaseModel](model: type[T], url: str, error: str) -> RakeResult[T]:
+    return RakeResult[model](url=url, items=[], request_ids=[], errors=[error])  # type: ignore[valid-type]
+
+
+def _format_validation_errors(error: ValidationError) -> list[str]:
+    return [f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" for err in error.errors()]
+
+
+def _rake_page[T: BaseModel](
+    url: str,
+    model: type[T],
+    targets: list[ExtractionTarget],
+    fetcher: Fetcher,
+    client: JevClient | None,
+    min_confidence: float,
+    on_request: RequestHook | None,
+) -> RakeResult[T]:
+    try:
+        html = fetcher.fetch(url)
+    except FetchError as e:
+        return _build_failed_result(model, url, str(e))
+
+    page = dom.parse_page(html)
+    blocks = collect.find_item_blocks(page, targets)
+    numbered: list[tuple[int | None, ItemBlock]] = (
+        [(None, blocks[0])] if blocks[0].is_whole_page else list(enumerate(blocks))
+    )
+
+    items: list[RakeItem[T]] = []
+    request_ids: list[str] = []
+    errors: list[str] = []
+    for start in range(0, len(numbered), MAX_ITEMS_PER_REQUEST):
+        chunk = numbered[start : start + MAX_ITEMS_PER_REQUEST]
+        candidates_by_item = {item: block.candidates for item, block in chunk}
+        request = jev.build_jev_request(url, page, targets, candidates_by_item)
+        if on_request is not None:
+            on_request(url, jev.build_jev_payload(request, MODEL))
+        if client is None:
+            continue
+
+        response = JevResponse()
+        if request.questions:
+            try:
+                response = client.ask(request.state, request.questions)
+            except JevAuthError:
+                raise
+            except JevError as e:
+                first, last = chunk[0][0], chunk[-1][0]
+                errors.append(str(e) if first is None else f"items {first}-{last}: {e}")
+                continue
+            if response.request_id:
+                request_ids.append(response.request_id)
+
+        for item, _ in chunk:
+            prefix = "" if item is None else f"item {item}: "
+            outcome = jev.parse_jev_response(request, response, targets, min_confidence, item)
+            if outcome.errors:
+                errors.extend(prefix + e for e in outcome.errors)
+                continue
+            try:
+                data = model.model_validate(outcome.values)
+            except ValidationError as e:
+                errors.extend(prefix + m for m in _format_validation_errors(e))
+                continue
+            items.append(RakeItem[model](data=data, fields=outcome.fields))  # type: ignore[valid-type]
+
+    return RakeResult[model](url=url, items=items, request_ids=request_ids, errors=errors)  # type: ignore[valid-type]
+
+
+def _rake_page_safely[T: BaseModel](url: str, model: type[T], *args: Any) -> RakeResult[T]:
+    try:
+        return _rake_page(url, model, *args)
+    except JevAuthError:
+        raise
+    except Exception as e:  # one bad page must not stop the others
+        return _build_failed_result(model, url, f"{type(e).__name__}: {e}")
+
+
+def rake[T: BaseModel](
+    urls: Sequence[str],
+    model: type[T],
+    *,
+    min_confidence: float = DEFAULT_MIN_CONFIDENCE,
+    fetcher: Fetcher | None = None,
+    client: JevClient | None = None,
+    dry_run: bool = False,
+    on_request: RequestHook | None = None,
+) -> list[RakeResult[T]]:
+    """Extract records of ``model`` from each URL, one page at a time, in the order of ``urls``.
+
+    Raises :class:`JevAuthError` if the API key is missing or rejected.
+    """
+    targets = build_targets_from_model(model)
+
+    created: list[HttpxFetcher | TypeSafeJevClient] = []
+    if fetcher is None:
+        fetcher = HttpxFetcher()
+        created.append(fetcher)
+    if dry_run:
+        client = None
+    elif client is None:
+        client = TypeSafeJevClient()
+        created.append(client)
+
+    try:
+        return [
+            _rake_page_safely(url, model, targets, fetcher, client, min_confidence, on_request)
+            for url in urls
+        ]
+    finally:
+        for resource in created:
+            resource.close()

jevrake-0.1.0/src/jevrake/_internal/client.py

@@ -0,0 +1,81 @@
+"""Send the question to Jev (TypeSafe AI API) and return its answer.
+
+Wraps the official SDK so the rest of jevrake does not depend on SDK types or exceptions.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Protocol
+
+import typesafe_sdk as ts
+
+from .exceptions import JevAuthError, JevError, MissingAPIKeyError
+from .models import JevChoiceAnswer, JevResponse
+
+if TYPE_CHECKING:
+    import httpx2
+
+API_KEY_ENV = "TYPESAFE_API_KEY"
+MODEL = "jev-latest"
+TIMEOUT = 30.0
+MAX_RETRIES = 3
+
+MISSING_API_KEY_MESSAGE = f"""\
+{API_KEY_ENV} is not set.
+Create an API key at https://console.typesafe.ai and set it with one of:
+  export {API_KEY_ENV}=<your API key>
+  cp .envrc.example .envrc  # fill in the key, then run `direnv allow`"""
+
+
+class JevClient(Protocol):
+    def ask(self, state: Mapping[str, Any], questions: Mapping[str, Any]) -> JevResponse: ...
+
+
+class TypeSafeJevClient:
+    """:class:`JevClient` backed by ``typesafe_sdk.TypeSafeClient``.
+
+    429 / 5xx responses are retried with exponential backoff (up to ``MAX_RETRIES`` times) by
+    the SDK's retry policy.
+    """
+
+    def __init__(self, *, transport: httpx2.BaseTransport | None = None) -> None:
+        api_key = os.environ.get(API_KEY_ENV)
+
+        if not api_key:
+            raise MissingAPIKeyError(MISSING_API_KEY_MESSAGE)
+
+        self._client = ts.TypeSafeClient(
+            api_key=api_key,
+            retry=ts.RetryPolicy(max_retries=MAX_RETRIES, timeout=None),
+            timeout=TIMEOUT,
+            transport=transport,
+        )
+
+    def ask(self, state: Mapping[str, Any], questions: Mapping[str, Any]) -> JevResponse:
+        try:
+            response = self._client.system_one(state=dict(state), questions=questions, model=MODEL)
+        except (ts.TypeSafeAuthenticationError, ts.TypeSafePermissionDeniedError) as e:
+            raise JevAuthError(
+                f"Jev API rejected the API key (HTTP {e.status}). Check {API_KEY_ENV}."
+            ) from None
+        except ts.TypeSafeAPITimeoutError:
+            raise JevError("Jev API request timed out") from None
+        except ts.TypeSafeAPIError as e:
+            raise JevError(f"Jev API error (HTTP {e.status}): {e.body}") from None
+        except ts.TypeSafeError as e:
+            raise JevError(f"Jev API error: {e}") from None
+
+        return JevResponse(
+            choices={
+                name: JevChoiceAnswer(
+                    choice=a.choice, confidence=a.confidence, probabilities=dict(a.probabilities)
+                )
+                for name, a in response.choices.items()
+            },
+            request_id=response.request_id,
+        )
+
+    def close(self) -> None:
+        self._client.close()