opencraw 0.0.2__py3-none-any.whl

opencraw/__init__.py

@@ -0,0 +1,33 @@
+"""Build OpenCraw recipes in Python, drive the real CLI, and write hooks. See README.md."""
+
+from .build_recipe_algorithm import input_recipe, output_recipe, step
+from .hook_pending_error import HookPending
+from .opencraw_cli_client import cli_command, run, validate
+from .opencraw_contract import (
+    HookContext,
+    HookRequest,
+    Recipe,
+    RecipeSource,
+    Validation,
+)
+from .opencraw_error import OpenCrawError
+from .serve_hook_use_case import serve_hook
+from .write_recipes_use_case import write_recipes
+
+__all__ = [
+    "HookContext",
+    "HookPending",
+    "HookRequest",
+    "OpenCrawError",
+    "Recipe",
+    "RecipeSource",
+    "Validation",
+    "cli_command",
+    "input_recipe",
+    "output_recipe",
+    "run",
+    "serve_hook",
+    "step",
+    "validate",
+    "write_recipes",
+]

opencraw/build_recipe_algorithm.py

@@ -0,0 +1,52 @@
+"""Plain-dict constructors for recipes. They save typing and nothing more: the step and field shapes are
+the engine's, so a key goes in exactly as the recipe guide spells it (``forEach``, ``inStock``) and the
+engine, not this module, says whether it is valid."""
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from .opencraw_contract import Recipe
+
+StartPoint = str | Mapping[str, Any]
+
+
+def step(step_type: str, **fields: Any) -> Recipe:
+    """One step: ``step('goto', url='{{start.url}}')`` is ``{'type': 'goto', 'url': '{{start.url}}'}``."""
+    return {"type": step_type, **fields}
+
+
+def output_recipe(
+    recipe_id: str, fields: Mapping[str, Any], version: int = 1, **extra: Any
+) -> Recipe:
+    """An output recipe: the records' schema."""
+    return {
+        "kind": "output",
+        "id": recipe_id,
+        "version": version,
+        "fields": dict(fields),
+        **extra,
+    }
+
+
+def input_recipe(
+    recipe_id: str,
+    output: str,
+    mode: str,
+    start: Sequence[StartPoint],
+    steps: Sequence[Mapping[str, Any]],
+    mapping: Mapping[str, Any] | None = None,
+    **extra: Any,
+) -> Recipe:
+    """An input recipe. A start given as a string is a URL: ``'https://a.example'`` is ``{'url': ...}``."""
+    return {
+        "kind": "input",
+        "id": recipe_id,
+        "output": output,
+        "mode": mode,
+        "start": [
+            {"url": point} if isinstance(point, str) else dict(point) for point in start
+        ],
+        "steps": [dict(each) for each in steps],
+        "mapping": dict(mapping or {}),
+        **extra,
+    }

opencraw/hook_pending_error.py

@@ -0,0 +1,12 @@
+class HookPending(Exception):
+    """Raised by a hook handler whose work is not done yet.
+
+    ``serve_hook`` turns it into a ``pending`` answer: OpenCraw runs the hook again, after
+    ``retry_after_ms`` (a second when it is ``None``), with the same ``idempotencyKey``, until the
+    handler returns or the hook's ``maxWaitMs`` passes. Start the work the first time, and look it up
+    by ``request["idempotencyKey"]`` on the later calls.
+    """
+
+    def __init__(self, retry_after_ms: int | None = None) -> None:
+        super().__init__("pending")
+        self.retry_after_ms = retry_after_ms

opencraw/opencraw_cli_client.py

@@ -0,0 +1,82 @@
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import tempfile
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from .opencraw_contract import RecipeSource, Validation
+from .opencraw_error import OpenCrawError
+from .write_recipes_use_case import write_recipes
+
+
+def cli_command() -> list[str]:
+    """How to start the CLI: ``OPENCRAW_CLI`` if set, else ``opencraw`` on the PATH, else ``npx``."""
+    configured = os.environ.get("OPENCRAW_CLI")
+    if configured:
+        return shlex.split(configured)
+    installed = shutil.which("opencraw")
+    if installed:
+        return [installed]
+    npx = shutil.which("npx")
+    if npx is None:
+        raise OpenCrawError(
+            "opencraw is not installed: run `npm i -g @opencraw/cli` or set OPENCRAW_CLI"
+        )
+    return [npx, "--yes", "@opencraw/cli"]
+
+
+def validate(recipes: Sequence[RecipeSource]) -> Validation:
+    """Runs ``opencraw validate``. The result is not an exception when the recipes are wrong: ``ok`` says so
+    and ``output`` lists every problem with its path."""
+    with _RecipePaths(recipes) as paths:
+        done = _call("validate", *paths)
+    return Validation(
+        ok=done.returncode == 0, output=(done.stdout + done.stderr).strip()
+    )
+
+
+def run(recipes: Sequence[RecipeSource], *options: str) -> list[dict[str, Any]]:
+    """Runs ``opencraw run`` and returns the records it printed as dicts. ``options`` are CLI flags, such as
+    ``'--only', 'shop-a'``. Raises ``OpenCrawError`` with the CLI's message when the crawl fails."""
+    with _RecipePaths(recipes) as paths:
+        done = _call("run", *paths, *options)
+    if done.returncode != 0:
+        raise OpenCrawError((done.stderr or done.stdout).strip())
+    return [
+        json.loads(line)
+        for line in done.stdout.splitlines()
+        if line.strip().startswith("{")
+    ]
+
+
+class _RecipePaths:
+    """Recipe paths for the CLI: dicts are written to a temporary folder that goes away afterwards."""
+
+    def __init__(self, recipes: Sequence[RecipeSource]) -> None:
+        self._recipes = recipes
+        self._folder = tempfile.TemporaryDirectory(prefix="opencraw-")
+
+    def __enter__(self) -> list[str]:
+        paths = [str(item) for item in self._recipes if not isinstance(item, Mapping)]
+        objects = [item for item in self._recipes if isinstance(item, Mapping)]
+        if objects:
+            paths.append(str(Path(self._folder.name)))
+            write_recipes(self._folder.name, objects)
+        return paths
+
+    def __exit__(self, *_: object) -> None:
+        self._folder.cleanup()
+
+
+def _call(*arguments: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        [*cli_command(), *arguments],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        check=False,
+    )

opencraw/opencraw_contract.py

@@ -0,0 +1,36 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, NotRequired, TypedDict
+
+# A recipe as a plain dict, in the engine's own shape.
+Recipe = dict[str, Any]
+
+# A recipe given to validate() or run(): a file or folder path, or the recipe as a dict.
+RecipeSource = str | Path | Mapping[str, Any]
+
+
+class HookContext(TypedDict):
+    """Where a hook runs: the recipe, and the extracted values visible at the call site."""
+
+    recipeId: str
+    scope: dict[str, Any]
+
+
+class HookRequest(TypedDict):
+    """The request OpenCraw sends a hook (``schemas/callout-request.schema.json``)."""
+
+    kind: str
+    name: str
+    input: NotRequired[Any]
+    args: dict[str, Any]
+    context: HookContext
+    idempotencyKey: str
+
+
+@dataclass(frozen=True)
+class Validation:
+    """What ``opencraw validate`` said: ``ok``, and ``output`` listing every problem with its path."""
+
+    ok: bool
+    output: str

opencraw/opencraw_error.py

@@ -0,0 +1,2 @@
+class OpenCrawError(RuntimeError):
+    """The CLI exited with an error; the message is what it printed."""

opencraw/serve_hook_use_case.py

@@ -0,0 +1,40 @@
+import json
+import sys
+from collections.abc import Callable
+from typing import Any, TextIO
+
+from .hook_pending_error import HookPending
+from .opencraw_contract import HookRequest
+
+
+def serve_hook(
+    handler: Callable[[Any, dict[str, Any], HookRequest], Any],
+    stdin: TextIO | None = None,
+    stdout: TextIO | None = None,
+) -> None:
+    """Answer one OpenCraw hook call: read the request, run ``handler``, write the answer.
+
+    ``handler(input, args, request)`` gets the value a hook transform is working on (``None`` for a
+    ``hook`` step), the recipe's arguments, and the whole request (``context``, ``idempotencyKey``).
+    What it returns is the output, and an exception becomes an ``error`` answer with its message, so
+    the step fails with that reason instead of a stack trace. Raising ``HookPending`` answers
+    ``pending`` instead, and OpenCraw asks again later. Use it as the whole of a hook script,
+    the program ``commandHook`` runs::
+
+        from opencraw import serve_hook
+
+        serve_hook(lambda value, args, request: str(value).strip().lower())
+    """
+    request: HookRequest = json.load(stdin or sys.stdin)
+    try:
+        answer: dict[str, Any] = {
+            "status": "ok",
+            "output": handler(request.get("input"), request.get("args", {}), request),
+        }
+    except HookPending as pending:
+        answer = {"status": "pending"}
+        if pending.retry_after_ms is not None:
+            answer["retryAfterMs"] = pending.retry_after_ms
+    except Exception as error:  # noqa: BLE001 - whatever the handler raises is the hook's answer
+        answer = {"status": "error", "error": str(error) or type(error).__name__}
+    json.dump(answer, stdout or sys.stdout)

opencraw/write_recipes_use_case.py

@@ -0,0 +1,24 @@
+import json
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+
+
+def write_recipes(
+    folder: str | Path, recipes: Iterable[Mapping[str, Any]]
+) -> list[Path]:
+    """Writes each recipe as ``<id>.input.json`` or ``<id>.output.json``, the layout the CLI, the Studio
+    and the MCP server read. The folder is created when missing. Returns the files written."""
+    target = Path(folder)
+    target.mkdir(parents=True, exist_ok=True)
+    written = []
+    for recipe in recipes:
+        kind = recipe.get("kind")
+        if kind not in ("input", "output") or not isinstance(recipe.get("id"), str):
+            raise ValueError(
+                f"a recipe needs a kind of 'input' or 'output' and a string id, got {dict(recipe)!r}"
+            )
+        path = target / f"{recipe['id']}.{kind}.json"
+        path.write_text(json.dumps(recipe, indent=2) + "\n", encoding="utf-8")
+        written.append(path)
+    return written

opencraw-0.0.2.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.5
+Name: opencraw
+Version: 0.0.2
+Summary: Build OpenCraw recipes in Python, drive the real OpenCraw CLI, and write hooks in Python.
+Project-URL: Homepage, https://github.com/Open-Craw/OpenCraw
+Project-URL: Repository, https://github.com/Open-Craw/OpenCraw
+Project-URL: Issues, https://github.com/Open-Craw/OpenCraw/issues
+License-Expression: MIT
+Keywords: crawler,opencraw,recipes,scraping
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# opencraw (Python)
+
+Build [OpenCraw](https://github.com/Open-Craw/OpenCraw) recipes in Python, and drive the real CLI. Standard library only; it
+needs Python 3.12+ and, to validate or run, the `opencraw` CLI (`npm i -g @opencraw/cli`, or
+`OPENCRAW_CLI` set to the command that starts it; with neither, `npx @opencraw/cli` is used).
+
+A recipe stays a plain dict in the engine's own shape, so the recipe guide
+([docs/recipes/authoring.md](https://github.com/Open-Craw/OpenCraw/blob/main/docs/recipes/authoring.md)) is the reference for every key. The constructors only save typing, and
+the engine, not this package, says what is valid.
+
+```python
+from opencraw import input_recipe, output_recipe, run, step, validate
+
+product = output_recipe(
+    "product",
+    {
+        "url": {"type": "url", "required": True, "key": True},
+        "title": {"type": "string", "required": True},
+    },
+)
+
+shops = [
+    input_recipe(
+        f"shop-{name}",
+        "product",
+        "web",
+        [url],
+        [
+            step("goto", url="{{start.url}}"),
+            step("extract", id="title", selector=selector, kind="css", take="text"),
+            step("emit"),
+        ],
+        {"url": {"from": "start.url"}, "title": {"from": "title"}},
+    )
+    for name, url, selector in [
+        ("a", "https://a.example", "h1"),
+        ("b", "https://b.example", ".name"),
+    ]
+]
+
+result = validate(
+    [product, *shops]
+)  # Validation(ok, output): problems are data, not exceptions
+records = run(
+    [product, *shops], "--only", "shop-a"
+)  # records as dicts; OpenCrawError if the crawl fails
+```
+
+Reserved words: pass a step's `as` or `from` through `**{"as": "json"}`.
+
+`write_recipes(folder, recipes)` writes `<id>.input.json` and `<id>.output.json` for the CLI, the Studio
+and the MCP server.
+
+## A hook written in Python
+
+A recipe's hook (`{ "op": "hook", "name": "slug" }`) can be a Python script. The hooks module registers it
+with `commandHook` (`@opencraw/core`), and the script is a few lines:
+
+```python
+# hooks/slug.py
+import re
+from opencraw import serve_hook
+
+serve_hook(
+    lambda value, args, request: re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip(
+        "-"
+    )
+)
+```
+
+`serve_hook(handler)` reads the request from stdin, calls `handler(input, args, request)` (`input` is `None`
+for a `hook` step; `request` carries `context` and `idempotencyKey`) and writes the answer to stdout. What the
+handler returns is the output; an exception becomes an error answer with its message, so the step fails with
+that reason. A handler whose work takes longer raises `HookPending(retry_after_ms)`: OpenCraw runs the script
+again after that wait, with the same `idempotencyKey`, until it returns or the hook's `maxWaitMs` passes, so the
+script starts the work once and looks it up by that key on later runs. The request and answer shapes are in
+`packages/core/schemas/callout-*.schema.json`.
+
+Not in this package: a Python engine and generated typed models. The crawl itself always runs in the
+OpenCraw CLI. It is published to PyPI as `opencraw`: `mnci ci release` versions and publishes what sits
+under `python-packages/`.
+
+## Development
+
+`nx test opencraw` (pytest; the CLI integration tests use this repository's built CLI and skip without it),
+`nx lint opencraw` (Ruff), `nx typecheck opencraw` (mypy strict).

opencraw-0.0.2.dist-info/RECORD

@@ -0,0 +1,12 @@
+opencraw/__init__.py,sha256=N6weBp2Nm2SU3BB-5np0A6xirQpzThEaxupyMfJZSq8,801
+opencraw/build_recipe_algorithm.py,sha256=813fbozrIemt9fRhH-GpU2k47qnEieCiPyzKsuYPQeA,1591
+opencraw/hook_pending_error.py,sha256=Dqntt243wcYetgSQWkbDuuLC9HUpKAtZz9FywVtPT4g,597
+opencraw/opencraw_cli_client.py,sha256=mctjvpYOlggUonCI-nnmvgQ-ytE4UATeUE1Y3abR45c,2854
+opencraw/opencraw_contract.py,sha256=kbHc2QFEx041jR-Zm4KkoI6e8_vTytS0FSeTftW9oAo,954
+opencraw/opencraw_error.py,sha256=MsoOkViFtUauCzJn8uDhM1EHoPusPFbbYK4dX6Z1y38,107
+opencraw/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+opencraw/serve_hook_use_case.py,sha256=Z0uL4vacBWcA0Y60Y8RbkyzO4UpZ6Uuk1ZmOPFm03UQ,1717
+opencraw/write_recipes_use_case.py,sha256=rxV7KFR0elcqNAj8KKXaw8hEP2g7Mg08vI2hMRONyWA,967
+opencraw-0.0.2.dist-info/METADATA,sha256=DCsdSrSpuGqD5xhq3xi-WKPogmKqr0VoTO9ORq_htuI,3800
+opencraw-0.0.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+opencraw-0.0.2.dist-info/RECORD,,

opencraw-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any