opencraw 0.0.2__tar.gz

opencraw-0.0.2/.gitignore

@@ -0,0 +1,70 @@
+# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files for more about ignoring files.
+
+# compiled output
+dist
+tmp
+out-tsc
+
+# dependencies
+node_modules
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+# misc
+/.sass-cache
+/connect.lock
+/coverage
+/libpeerconnection.log
+npm-debug.log
+yarn-error.log
+testem.log
+/typings
+
+# System Files
+.DS_Store
+Thumbs.db
+
+.nx/cache
+.nx/workspace-data
+.cursor/rules/nx-rules.mdc
+.github/instructions/nx.instructions.md
+
+.claude/worktrees
+.claude/settings.local.json
+.nx/polygraph
+.nx/self-healing
+.nx/migrate-runs
+
+# Added by MoNecromanCI: `npm run format`/`lint` run `eslint --cache`.
+.eslintcache
+
+test-output
+
+# persistent browser profiles (logins, cookies): never commit
+.opencraw/
+
+vite.config.*.timestamp*
+
+# MoNecromanCI: keep the committed per-environment Vite config (public VITE_ values)
+!apps/*/.env.dev
+!apps/*/.env.uat
+!apps/*/.env.prod
+
+# Added by MoNecromanCI: Python bytecode. The tool caches (.mypy_cache,
+# .pytest_cache, .ruff_cache) write their own .gitignore; CPython does not.
+__pycache__/
+*.py[cod]

opencraw-0.0.2/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.5
+Name: opencraw
+Version: 0.0.2
+Summary: Build OpenCraw recipes in Python, drive the real OpenCraw CLI, and write hooks in Python.
+Project-URL: Homepage, https://github.com/Open-Craw/OpenCraw
+Project-URL: Repository, https://github.com/Open-Craw/OpenCraw
+Project-URL: Issues, https://github.com/Open-Craw/OpenCraw/issues
+License-Expression: MIT
+Keywords: crawler,opencraw,recipes,scraping
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# opencraw (Python)
+
+Build [OpenCraw](https://github.com/Open-Craw/OpenCraw) recipes in Python, and drive the real CLI. Standard library only; it
+needs Python 3.12+ and, to validate or run, the `opencraw` CLI (`npm i -g @opencraw/cli`, or
+`OPENCRAW_CLI` set to the command that starts it; with neither, `npx @opencraw/cli` is used).
+
+A recipe stays a plain dict in the engine's own shape, so the recipe guide
+([docs/recipes/authoring.md](https://github.com/Open-Craw/OpenCraw/blob/main/docs/recipes/authoring.md)) is the reference for every key. The constructors only save typing, and
+the engine, not this package, says what is valid.
+
+```python
+from opencraw import input_recipe, output_recipe, run, step, validate
+
+product = output_recipe(
+    "product",
+    {
+        "url": {"type": "url", "required": True, "key": True},
+        "title": {"type": "string", "required": True},
+    },
+)
+
+shops = [
+    input_recipe(
+        f"shop-{name}",
+        "product",
+        "web",
+        [url],
+        [
+            step("goto", url="{{start.url}}"),
+            step("extract", id="title", selector=selector, kind="css", take="text"),
+            step("emit"),
+        ],
+        {"url": {"from": "start.url"}, "title": {"from": "title"}},
+    )
+    for name, url, selector in [
+        ("a", "https://a.example", "h1"),
+        ("b", "https://b.example", ".name"),
+    ]
+]
+
+result = validate(
+    [product, *shops]
+)  # Validation(ok, output): problems are data, not exceptions
+records = run(
+    [product, *shops], "--only", "shop-a"
+)  # records as dicts; OpenCrawError if the crawl fails
+```
+
+Reserved words: pass a step's `as` or `from` through `**{"as": "json"}`.
+
+`write_recipes(folder, recipes)` writes `<id>.input.json` and `<id>.output.json` for the CLI, the Studio
+and the MCP server.
+
+## A hook written in Python
+
+A recipe's hook (`{ "op": "hook", "name": "slug" }`) can be a Python script. The hooks module registers it
+with `commandHook` (`@opencraw/core`), and the script is a few lines:
+
+```python
+# hooks/slug.py
+import re
+from opencraw import serve_hook
+
+serve_hook(
+    lambda value, args, request: re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip(
+        "-"
+    )
+)
+```
+
+`serve_hook(handler)` reads the request from stdin, calls `handler(input, args, request)` (`input` is `None`
+for a `hook` step; `request` carries `context` and `idempotencyKey`) and writes the answer to stdout. What the
+handler returns is the output; an exception becomes an error answer with its message, so the step fails with
+that reason. A handler whose work takes longer raises `HookPending(retry_after_ms)`: OpenCraw runs the script
+again after that wait, with the same `idempotencyKey`, until it returns or the hook's `maxWaitMs` passes, so the
+script starts the work once and looks it up by that key on later runs. The request and answer shapes are in
+`packages/core/schemas/callout-*.schema.json`.
+
+Not in this package: a Python engine and generated typed models. The crawl itself always runs in the
+OpenCraw CLI. It is published to PyPI as `opencraw`: `mnci ci release` versions and publishes what sits
+under `python-packages/`.
+
+## Development
+
+`nx test opencraw` (pytest; the CLI integration tests use this repository's built CLI and skip without it),
+`nx lint opencraw` (Ruff), `nx typecheck opencraw` (mypy strict).

opencraw-0.0.2/README.md

@@ -0,0 +1,86 @@
+# opencraw (Python)
+
+Build [OpenCraw](https://github.com/Open-Craw/OpenCraw) recipes in Python, and drive the real CLI. Standard library only; it
+needs Python 3.12+ and, to validate or run, the `opencraw` CLI (`npm i -g @opencraw/cli`, or
+`OPENCRAW_CLI` set to the command that starts it; with neither, `npx @opencraw/cli` is used).
+
+A recipe stays a plain dict in the engine's own shape, so the recipe guide
+([docs/recipes/authoring.md](https://github.com/Open-Craw/OpenCraw/blob/main/docs/recipes/authoring.md)) is the reference for every key. The constructors only save typing, and
+the engine, not this package, says what is valid.
+
+```python
+from opencraw import input_recipe, output_recipe, run, step, validate
+
+product = output_recipe(
+    "product",
+    {
+        "url": {"type": "url", "required": True, "key": True},
+        "title": {"type": "string", "required": True},
+    },
+)
+
+shops = [
+    input_recipe(
+        f"shop-{name}",
+        "product",
+        "web",
+        [url],
+        [
+            step("goto", url="{{start.url}}"),
+            step("extract", id="title", selector=selector, kind="css", take="text"),
+            step("emit"),
+        ],
+        {"url": {"from": "start.url"}, "title": {"from": "title"}},
+    )
+    for name, url, selector in [
+        ("a", "https://a.example", "h1"),
+        ("b", "https://b.example", ".name"),
+    ]
+]
+
+result = validate(
+    [product, *shops]
+)  # Validation(ok, output): problems are data, not exceptions
+records = run(
+    [product, *shops], "--only", "shop-a"
+)  # records as dicts; OpenCrawError if the crawl fails
+```
+
+Reserved words: pass a step's `as` or `from` through `**{"as": "json"}`.
+
+`write_recipes(folder, recipes)` writes `<id>.input.json` and `<id>.output.json` for the CLI, the Studio
+and the MCP server.
+
+## A hook written in Python
+
+A recipe's hook (`{ "op": "hook", "name": "slug" }`) can be a Python script. The hooks module registers it
+with `commandHook` (`@opencraw/core`), and the script is a few lines:
+
+```python
+# hooks/slug.py
+import re
+from opencraw import serve_hook
+
+serve_hook(
+    lambda value, args, request: re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip(
+        "-"
+    )
+)
+```
+
+`serve_hook(handler)` reads the request from stdin, calls `handler(input, args, request)` (`input` is `None`
+for a `hook` step; `request` carries `context` and `idempotencyKey`) and writes the answer to stdout. What the
+handler returns is the output; an exception becomes an error answer with its message, so the step fails with
+that reason. A handler whose work takes longer raises `HookPending(retry_after_ms)`: OpenCraw runs the script
+again after that wait, with the same `idempotencyKey`, until it returns or the hook's `maxWaitMs` passes, so the
+script starts the work once and looks it up by that key on later runs. The request and answer shapes are in
+`packages/core/schemas/callout-*.schema.json`.
+
+Not in this package: a Python engine and generated typed models. The crawl itself always runs in the
+OpenCraw CLI. It is published to PyPI as `opencraw`: `mnci ci release` versions and publishes what sits
+under `python-packages/`.
+
+## Development
+
+`nx test opencraw` (pytest; the CLI integration tests use this repository's built CLI and skip without it),
+`nx lint opencraw` (Ruff), `nx typecheck opencraw` (mypy strict).

opencraw-0.0.2/opencraw/__init__.py

@@ -0,0 +1,33 @@
+"""Build OpenCraw recipes in Python, drive the real CLI, and write hooks. See README.md."""
+
+from .build_recipe_algorithm import input_recipe, output_recipe, step
+from .hook_pending_error import HookPending
+from .opencraw_cli_client import cli_command, run, validate
+from .opencraw_contract import (
+    HookContext,
+    HookRequest,
+    Recipe,
+    RecipeSource,
+    Validation,
+)
+from .opencraw_error import OpenCrawError
+from .serve_hook_use_case import serve_hook
+from .write_recipes_use_case import write_recipes
+
+__all__ = [
+    "HookContext",
+    "HookPending",
+    "HookRequest",
+    "OpenCrawError",
+    "Recipe",
+    "RecipeSource",
+    "Validation",
+    "cli_command",
+    "input_recipe",
+    "output_recipe",
+    "run",
+    "serve_hook",
+    "step",
+    "validate",
+    "write_recipes",
+]

opencraw-0.0.2/opencraw/build_recipe_algorithm.py

@@ -0,0 +1,52 @@
+"""Plain-dict constructors for recipes. They save typing and nothing more: the step and field shapes are
+the engine's, so a key goes in exactly as the recipe guide spells it (``forEach``, ``inStock``) and the
+engine, not this module, says whether it is valid."""
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from .opencraw_contract import Recipe
+
+StartPoint = str | Mapping[str, Any]
+
+
+def step(step_type: str, **fields: Any) -> Recipe:
+    """One step: ``step('goto', url='{{start.url}}')`` is ``{'type': 'goto', 'url': '{{start.url}}'}``."""
+    return {"type": step_type, **fields}
+
+
+def output_recipe(
+    recipe_id: str, fields: Mapping[str, Any], version: int = 1, **extra: Any
+) -> Recipe:
+    """An output recipe: the records' schema."""
+    return {
+        "kind": "output",
+        "id": recipe_id,
+        "version": version,
+        "fields": dict(fields),
+        **extra,
+    }
+
+
+def input_recipe(
+    recipe_id: str,
+    output: str,
+    mode: str,
+    start: Sequence[StartPoint],
+    steps: Sequence[Mapping[str, Any]],
+    mapping: Mapping[str, Any] | None = None,
+    **extra: Any,
+) -> Recipe:
+    """An input recipe. A start given as a string is a URL: ``'https://a.example'`` is ``{'url': ...}``."""
+    return {
+        "kind": "input",
+        "id": recipe_id,
+        "output": output,
+        "mode": mode,
+        "start": [
+            {"url": point} if isinstance(point, str) else dict(point) for point in start
+        ],
+        "steps": [dict(each) for each in steps],
+        "mapping": dict(mapping or {}),
+        **extra,
+    }

opencraw-0.0.2/opencraw/hook_pending_error.py

@@ -0,0 +1,12 @@
+class HookPending(Exception):
+    """Raised by a hook handler whose work is not done yet.
+
+    ``serve_hook`` turns it into a ``pending`` answer: OpenCraw runs the hook again, after
+    ``retry_after_ms`` (a second when it is ``None``), with the same ``idempotencyKey``, until the
+    handler returns or the hook's ``maxWaitMs`` passes. Start the work the first time, and look it up
+    by ``request["idempotencyKey"]`` on the later calls.
+    """
+
+    def __init__(self, retry_after_ms: int | None = None) -> None:
+        super().__init__("pending")
+        self.retry_after_ms = retry_after_ms

opencraw-0.0.2/opencraw/opencraw_cli_client.py

@@ -0,0 +1,82 @@
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import tempfile
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from .opencraw_contract import RecipeSource, Validation
+from .opencraw_error import OpenCrawError
+from .write_recipes_use_case import write_recipes
+
+
+def cli_command() -> list[str]:
+    """How to start the CLI: ``OPENCRAW_CLI`` if set, else ``opencraw`` on the PATH, else ``npx``."""
+    configured = os.environ.get("OPENCRAW_CLI")
+    if configured:
+        return shlex.split(configured)
+    installed = shutil.which("opencraw")
+    if installed:
+        return [installed]
+    npx = shutil.which("npx")
+    if npx is None:
+        raise OpenCrawError(
+            "opencraw is not installed: run `npm i -g @opencraw/cli` or set OPENCRAW_CLI"
+        )
+    return [npx, "--yes", "@opencraw/cli"]
+
+
+def validate(recipes: Sequence[RecipeSource]) -> Validation:
+    """Runs ``opencraw validate``. The result is not an exception when the recipes are wrong: ``ok`` says so
+    and ``output`` lists every problem with its path."""
+    with _RecipePaths(recipes) as paths:
+        done = _call("validate", *paths)
+    return Validation(
+        ok=done.returncode == 0, output=(done.stdout + done.stderr).strip()
+    )
+
+
+def run(recipes: Sequence[RecipeSource], *options: str) -> list[dict[str, Any]]:
+    """Runs ``opencraw run`` and returns the records it printed as dicts. ``options`` are CLI flags, such as
+    ``'--only', 'shop-a'``. Raises ``OpenCrawError`` with the CLI's message when the crawl fails."""
+    with _RecipePaths(recipes) as paths:
+        done = _call("run", *paths, *options)
+    if done.returncode != 0:
+        raise OpenCrawError((done.stderr or done.stdout).strip())
+    return [
+        json.loads(line)
+        for line in done.stdout.splitlines()
+        if line.strip().startswith("{")
+    ]
+
+
+class _RecipePaths:
+    """Recipe paths for the CLI: dicts are written to a temporary folder that goes away afterwards."""
+
+    def __init__(self, recipes: Sequence[RecipeSource]) -> None:
+        self._recipes = recipes
+        self._folder = tempfile.TemporaryDirectory(prefix="opencraw-")
+
+    def __enter__(self) -> list[str]:
+        paths = [str(item) for item in self._recipes if not isinstance(item, Mapping)]
+        objects = [item for item in self._recipes if isinstance(item, Mapping)]
+        if objects:
+            paths.append(str(Path(self._folder.name)))
+            write_recipes(self._folder.name, objects)
+        return paths
+
+    def __exit__(self, *_: object) -> None:
+        self._folder.cleanup()
+
+
+def _call(*arguments: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        [*cli_command(), *arguments],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        check=False,
+    )

opencraw-0.0.2/opencraw/opencraw_contract.py

@@ -0,0 +1,36 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, NotRequired, TypedDict
+
+# A recipe as a plain dict, in the engine's own shape.
+Recipe = dict[str, Any]
+
+# A recipe given to validate() or run(): a file or folder path, or the recipe as a dict.
+RecipeSource = str | Path | Mapping[str, Any]
+
+
+class HookContext(TypedDict):
+    """Where a hook runs: the recipe, and the extracted values visible at the call site."""
+
+    recipeId: str
+    scope: dict[str, Any]
+
+
+class HookRequest(TypedDict):
+    """The request OpenCraw sends a hook (``schemas/callout-request.schema.json``)."""
+
+    kind: str
+    name: str
+    input: NotRequired[Any]
+    args: dict[str, Any]
+    context: HookContext
+    idempotencyKey: str
+
+
+@dataclass(frozen=True)
+class Validation:
+    """What ``opencraw validate`` said: ``ok``, and ``output`` listing every problem with its path."""
+
+    ok: bool
+    output: str

opencraw-0.0.2/opencraw/opencraw_error.py

@@ -0,0 +1,2 @@
+class OpenCrawError(RuntimeError):
+    """The CLI exited with an error; the message is what it printed."""

opencraw-0.0.2/opencraw/serve_hook_use_case.py

@@ -0,0 +1,40 @@
+import json
+import sys
+from collections.abc import Callable
+from typing import Any, TextIO
+
+from .hook_pending_error import HookPending
+from .opencraw_contract import HookRequest
+
+
+def serve_hook(
+    handler: Callable[[Any, dict[str, Any], HookRequest], Any],
+    stdin: TextIO | None = None,
+    stdout: TextIO | None = None,
+) -> None:
+    """Answer one OpenCraw hook call: read the request, run ``handler``, write the answer.
+
+    ``handler(input, args, request)`` gets the value a hook transform is working on (``None`` for a
+    ``hook`` step), the recipe's arguments, and the whole request (``context``, ``idempotencyKey``).
+    What it returns is the output, and an exception becomes an ``error`` answer with its message, so
+    the step fails with that reason instead of a stack trace. Raising ``HookPending`` answers
+    ``pending`` instead, and OpenCraw asks again later. Use it as the whole of a hook script,
+    the program ``commandHook`` runs::
+
+        from opencraw import serve_hook
+
+        serve_hook(lambda value, args, request: str(value).strip().lower())
+    """
+    request: HookRequest = json.load(stdin or sys.stdin)
+    try:
+        answer: dict[str, Any] = {
+            "status": "ok",
+            "output": handler(request.get("input"), request.get("args", {}), request),
+        }
+    except HookPending as pending:
+        answer = {"status": "pending"}
+        if pending.retry_after_ms is not None:
+            answer["retryAfterMs"] = pending.retry_after_ms
+    except Exception as error:  # noqa: BLE001 - whatever the handler raises is the hook's answer
+        answer = {"status": "error", "error": str(error) or type(error).__name__}
+    json.dump(answer, stdout or sys.stdout)

opencraw-0.0.2/opencraw/write_recipes_use_case.py

@@ -0,0 +1,24 @@
+import json
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+
+
+def write_recipes(
+    folder: str | Path, recipes: Iterable[Mapping[str, Any]]
+) -> list[Path]:
+    """Writes each recipe as ``<id>.input.json`` or ``<id>.output.json``, the layout the CLI, the Studio
+    and the MCP server read. The folder is created when missing. Returns the files written."""
+    target = Path(folder)
+    target.mkdir(parents=True, exist_ok=True)
+    written = []
+    for recipe in recipes:
+        kind = recipe.get("kind")
+        if kind not in ("input", "output") or not isinstance(recipe.get("id"), str):
+            raise ValueError(
+                f"a recipe needs a kind of 'input' or 'output' and a string id, got {dict(recipe)!r}"
+            )
+        path = target / f"{recipe['id']}.{kind}.json"
+        path.write_text(json.dumps(recipe, indent=2) + "\n", encoding="utf-8")
+        written.append(path)
+    return written

opencraw-0.0.2/project.json

@@ -0,0 +1,45 @@
+{
+  "name": "opencraw",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "projectType": "library",
+  "sourceRoot": "python-packages/opencraw",
+  "targets": {
+    "lint": {
+      "executor": "@mnci/nx-python-pip:lint",
+      "options": {}
+    },
+    "test": {
+      "executor": "@mnci/nx-python-pip:test",
+      "options": {},
+      "dependsOn": [
+        "^build"
+      ]
+    },
+    "typecheck": {
+      "executor": "@mnci/nx-python-pip:typecheck",
+      "options": {}
+    },
+    "build": {
+      "executor": "@mnci/nx-python-pip:build",
+      "outputs": [
+        "{projectRoot}/dist"
+      ],
+      "options": {}
+    },
+    "nx-release-publish": {
+      "executor": "@mnci/nx-python-pip:publish",
+      "dependsOn": [
+        "build"
+      ],
+      "options": {}
+    }
+  },
+  "implicitDependencies": [
+    "@opencraw/cli"
+  ],
+  "release": {
+    "version": {
+      "versionActions": "@mnci/nx-python-pip/release/version-actions"
+    }
+  }
+}

opencraw-0.0.2/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "opencraw"
+version = "0.0.2"
+description = "Build OpenCraw recipes in Python, drive the real OpenCraw CLI, and write hooks in Python."
+readme = "README.md"
+license = "MIT"
+keywords = ["opencraw", "crawler", "scraping", "recipes"]
+requires-python = ">=3.12"
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/Open-Craw/OpenCraw"
+Repository = "https://github.com/Open-Craw/OpenCraw"
+Issues = "https://github.com/Open-Craw/OpenCraw/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["opencraw"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.mypy]
+strict = true
+# Stub-less third-party packages, and ONLY those. Measured on a generated
+# project: plain strict passes out of the box, and the one thing that breaks it
+# in practice is importing a library that ships no type stubs - "import yaml"
+# fails as import-untyped on code you wrote normally and cannot fix. Disabling
+# that one code keeps every other strict check on your own code, and a module
+# that genuinely cannot be found still fails, as import-not-found.
+disable_error_code = ["import-untyped"]

opencraw-0.0.2/tests/test_build_recipe_algorithm.py

@@ -0,0 +1,39 @@
+import unittest
+
+from opencraw import input_recipe, output_recipe, step
+
+
+class BuildRecipeTest(unittest.TestCase):
+    def test_step_is_a_dict_with_its_type(self) -> None:
+        self.assertEqual(
+            step("goto", url="{{start.url}}"), {"type": "goto", "url": "{{start.url}}"}
+        )
+
+    def test_input_recipe_turns_a_string_start_into_a_url_point(self) -> None:
+        recipe = input_recipe(
+            "a",
+            "p",
+            "web",
+            ["https://a.example", {"url": "https://b.example"}],
+            [step("emit")],
+        )
+        self.assertEqual(
+            recipe["start"],
+            [{"url": "https://a.example"}, {"url": "https://b.example"}],
+        )
+        self.assertEqual(recipe["kind"], "input")
+        self.assertEqual(recipe["mapping"], {})
+
+    def test_extra_keys_pass_through_untouched(self) -> None:
+        recipe = input_recipe(
+            "a", "p", "web", ["u"], [], vars={"user": "demo"}, limits={"concurrency": 2}
+        )
+        self.assertEqual(recipe["vars"], {"user": "demo"})
+        self.assertEqual(recipe["limits"], {"concurrency": 2})
+
+    def test_output_recipe_defaults_to_version_one(self) -> None:
+        self.assertEqual(output_recipe("p", {"url": {"type": "url"}})["version"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

opencraw-0.0.2/tests/test_opencraw_cli_client.py

@@ -0,0 +1,92 @@
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from opencraw import (
+    OpenCrawError,
+    cli_command,
+    input_recipe,
+    output_recipe,
+    run,
+    step,
+    validate,
+)
+
+REPO_CLI = (
+    Path(__file__).resolve().parents[3] / "packages" / "cli" / "bin" / "opencraw.mjs"
+)
+CLI_BUILT = (REPO_CLI.parent.parent / "dist" / "index.esm.js").exists()
+
+
+class CliCommandTest(unittest.TestCase):
+    def test_the_environment_wins(self) -> None:
+        with mock.patch.dict(os.environ, {"OPENCRAW_CLI": "node /x/cli.mjs"}):
+            self.assertEqual(cli_command(), ["node", "/x/cli.mjs"])
+
+
+@unittest.skipUnless(CLI_BUILT, "the repository's CLI is not built")
+class CliIntegrationTest(unittest.TestCase):
+    """Against the repository's own CLI, through the files this package writes."""
+
+    def setUp(self) -> None:
+        patcher = mock.patch.dict(os.environ, {"OPENCRAW_CLI": f'node "{REPO_CLI}"'})
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        self.folder = tempfile.TemporaryDirectory()
+        self.addCleanup(self.folder.cleanup)
+        data = Path(self.folder.name) / "data.json"
+        data.write_text(
+            json.dumps({"items": [{"name": "Widget"}, {"name": "Gadget"}]}),
+            encoding="utf-8",
+        )
+        self.output = output_recipe(
+            "thing", {"name": {"type": "string", "required": True, "key": True}}
+        )
+        self.input = input_recipe(
+            "things",
+            "thing",
+            "api",
+            [data.as_uri()],
+            [
+                step("request", id="response", url="{{start.url}}", **{"as": "json"}),
+                step(
+                    "extract",
+                    id="items",
+                    **{"from": "response"},
+                    selector="$.items[*]",
+                    kind="jsonpath",
+                    take="json",
+                    many=True,
+                ),
+                step("forEach", over="items", **{"as": "item"}, emit=True, steps=[]),
+            ],
+            {"name": {"from": "item.name"}},
+        )
+
+    def test_validate_accepts_good_recipes(self) -> None:
+        result = validate([self.output, self.input])
+        self.assertTrue(result.ok, result.output)
+
+    def test_validate_reports_a_bad_step_without_raising(self) -> None:
+        bad = {**self.input, "steps": [step("gotoo", url="x")]}
+        result = validate([self.output, bad])
+        self.assertFalse(result.ok)
+        self.assertIn("things", result.output)
+
+    def test_run_returns_the_records_as_dicts(self) -> None:
+        records = run([self.output, self.input])
+        self.assertEqual(
+            sorted(record["name"] for record in records), ["Gadget", "Widget"]
+        )
+
+    def test_run_raises_with_the_cli_message_on_failure(self) -> None:
+        bad = {**self.input, "steps": [step("gotoo", url="x")]}
+        with self.assertRaises(OpenCrawError):
+            run([self.output, bad])
+
+
+if __name__ == "__main__":
+    unittest.main()

opencraw-0.0.2/tests/test_serve_hook_use_case.py

@@ -0,0 +1,112 @@
+import io
+import json
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+from typing import Any
+
+from opencraw import HookPending, HookRequest, serve_hook
+
+CORE = (
+    Path(__file__).resolve().parents[3] / "packages" / "core" / "dist" / "index.esm.js"
+)
+
+
+def answer_to(request: dict[str, Any], handler: Any) -> dict[str, Any]:
+    out = io.StringIO()
+    serve_hook(handler, io.StringIO(json.dumps(request)), out)
+    answer: dict[str, Any] = json.loads(out.getvalue())
+    return answer
+
+
+REQUEST = {
+    "kind": "hook",
+    "name": "slug",
+    "input": "Hello World",
+    "args": {"max": 5},
+    "context": {"recipeId": "books", "scope": {}},
+    "idempotencyKey": "abc",
+}
+
+
+class ServeHookTest(unittest.TestCase):
+    def test_the_return_value_is_the_output(self) -> None:
+        answer = answer_to(
+            REQUEST, lambda value, args, _request: f"{value}:{args['max']}"
+        )
+
+        self.assertEqual(answer, {"status": "ok", "output": "Hello World:5"})
+
+    def test_the_handler_sees_the_whole_request(self) -> None:
+        seen: list[HookRequest] = []
+
+        def handler(_value: Any, _args: dict[str, Any], request: HookRequest) -> None:
+            seen.append(request)
+
+        answer_to(REQUEST, handler)
+
+        self.assertEqual(seen[0]["context"]["recipeId"], "books")
+        self.assertEqual(seen[0]["idempotencyKey"], "abc")
+
+    def test_a_hook_step_has_no_input(self) -> None:
+        request = {key: value for key, value in REQUEST.items() if key != "input"}
+
+        answer = answer_to(request, lambda value, _args, _request: value is None)
+
+        self.assertEqual(answer["output"], True)
+
+    def test_an_exception_becomes_an_error_answer(self) -> None:
+        def handler(_value: Any, _args: dict[str, Any], _request: HookRequest) -> None:
+            raise ValueError("no such price")
+
+        self.assertEqual(
+            answer_to(REQUEST, handler), {"status": "error", "error": "no such price"}
+        )
+
+    def test_raising_hook_pending_answers_pending_with_the_wait(self) -> None:
+        def handler(_value: Any, _args: dict[str, Any], _request: HookRequest) -> None:
+            raise HookPending(250)
+
+        self.assertEqual(
+            answer_to(REQUEST, handler), {"status": "pending", "retryAfterMs": 250}
+        )
+
+    def test_hook_pending_without_a_wait_leaves_it_to_the_caller(self) -> None:
+        def handler(_value: Any, _args: dict[str, Any], _request: HookRequest) -> None:
+            raise HookPending()
+
+        self.assertEqual(answer_to(REQUEST, handler), {"status": "pending"})
+
+    def test_an_exception_without_a_message_names_its_type(self) -> None:
+        def handler(_value: Any, _args: dict[str, Any], _request: HookRequest) -> None:
+            raise KeyError
+
+        self.assertEqual(answer_to(REQUEST, handler)["error"], "KeyError")
+
+
+@unittest.skipUnless(CORE.exists(), "the repository's core is not built")
+class CommandHookIntegrationTest(unittest.TestCase):
+    """The real ``commandHook`` from the built core runs a script that uses ``serve_hook``."""
+
+    def test_a_python_script_is_a_hook(self) -> None:
+        package_root = Path(__file__).resolve().parents[1]
+        script = (
+            "from opencraw import serve_hook\n"
+            "serve_hook(lambda value, args, request: str(value).strip().upper() + '!' * args['bangs'])\n"
+        )
+        node = f"""
+        import {{ commandHook }} from {json.dumps(CORE.as_uri())}
+        const hook = commandHook('shout', [{json.dumps(sys.executable)}, '-c', {json.dumps(script)}], {{ cwd: {json.dumps(str(package_root))} }})
+        process.stdout.write(JSON.stringify(await hook('  hi ', {{ bangs: 2 }}, {{ recipeId: 'r', scope: {{}}, log: () => undefined }})))
+        """
+
+        result = subprocess.run(
+            ["node", "--input-type=module", "-e", node],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(json.loads(result.stdout), "HI!!")

opencraw-0.0.2/tests/test_write_recipes_use_case.py

@@ -0,0 +1,33 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from opencraw import input_recipe, output_recipe, write_recipes
+
+
+class WriteRecipesTest(unittest.TestCase):
+    def test_writes_each_recipe_under_its_id_and_kind(self) -> None:
+        with tempfile.TemporaryDirectory() as folder:
+            target = Path(folder) / "recipes"
+            files = write_recipes(
+                target,
+                [output_recipe("p", {}), input_recipe("a", "p", "web", ["u"], [])],
+            )
+            self.assertEqual(
+                sorted(file.name for file in files), ["a.input.json", "p.output.json"]
+            )
+            self.assertEqual(
+                json.loads((target / "a.input.json").read_text(encoding="utf-8"))[
+                    "output"
+                ],
+                "p",
+            )
+
+    def test_rejects_a_recipe_without_kind_or_id(self) -> None:
+        with tempfile.TemporaryDirectory() as folder, self.assertRaises(ValueError):
+            write_recipes(folder, [{"id": "x"}])
+
+
+if __name__ == "__main__":
+    unittest.main()