doc-control 0.0.1.dev0__py3-none-any.whl

doc_control/recipes/engine.py

@@ -0,0 +1,209 @@
+"""Run a validated recipe over paragraph texts and produce a batch proposal.
+
+The engine reads nothing but the texts it is given and returns operation dicts for
+``batch``; it writes nothing. Every address in the proposal refers to the document as
+read, so the result is a valid batch. With ``repeat_until_stable`` the recipe is run again
+on the text its own replacements would leave, until a pass changes nothing or
+``max_passes`` is reached. A paragraph that needed more than one pass is proposed as one
+``text.replace`` over the smallest range that differs between before and after.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from doc_control.model.address import Address
+from doc_control.model.search import IsolatedScan, Match, Matcher, SearchOptions
+from doc_control.ops.tracked import boundary_ok
+from doc_control.recipes.schema import DEFAULT_PASSES, SUBSTITUTION, search_options
+
+MAX_OPS = 2000
+TIME_BUDGET_SECONDS = 10.0
+
+
+@dataclass
+class Proposal:
+    ops: list[dict[str, Any]]
+    passes: int
+    stable: bool
+    matches: int
+    warnings: list[str] = field(default_factory=list)
+
+
+def _substitute(text: str, match: str, para: str, groups: Sequence[str]) -> str:
+    def one(m: Any) -> str:
+        ref = m.group(1)
+        if ref == "$":
+            return "$"
+        if ref == "match":
+            return match
+        if ref == "para":
+            return para
+        return groups[int(ref) - 1]
+
+    return SUBSTITUTION.sub(one, text)
+
+
+def _smallest_edit(before: str, after: str) -> tuple[int, int, str]:
+    """``(start, end, new)``: replace ``before[start:end]`` with ``new``."""
+    limit = min(len(before), len(after))
+    head = 0
+    while head < limit and before[head] == after[head]:
+        head += 1
+    while head and not (boundary_ok(before, head) and boundary_ok(after, head)):
+        head -= 1
+    tail = 0
+    while tail < limit - head and before[-1 - tail] == after[-1 - tail]:
+        tail += 1
+    while tail and not (
+        boundary_ok(before, len(before) - tail) and boundary_ok(after, len(after) - tail)
+    ):
+        tail -= 1
+    return head, len(before) - tail, after[head : len(after) - tail]
+
+
+def _scan(
+    options: SearchOptions, matcher: Matcher, items: list[tuple[int, str]], deadline: float
+) -> tuple[list[tuple[int, list[Match]]], int | None]:
+    """Matches per paragraph, and the paragraph the time budget stopped at (or ``None``).
+
+    A regex runs in a child process that is killed at the deadline (Python's regex engine
+    cannot be interrupted, and the static screen cannot catch every pattern that
+    backtracks badly). A literal cannot backtrack and is searched here.
+    """
+    if options.regex:
+        scan = IsolatedScan(options, items, deadline, detail=True)
+        with contextlib.closing(scan.__iter__()) as found:
+            return [(number, list(spans)) for number, spans in found], scan.stopped_at
+    out: list[tuple[int, list[Match]]] = []
+    for done, (number, text) in enumerate(items):
+        if done and time.monotonic() > deadline:
+            return out, number
+        spans = list(matcher.matches(text))
+        if spans:
+            out.append((number, spans))
+    return out, None
+
+
+def propose(
+    recipe: dict[str, Any],
+    paragraphs: Sequence[tuple[Address, str]],
+    *,
+    deadline: float | None = None,
+) -> Proposal:
+    """The batch proposal of ``recipe`` (already validated) over ``paragraphs``."""
+    options = search_options(recipe)
+    matcher = Matcher(options)
+    steps: list[dict[str, Any]] = recipe["propose"]
+    repeat = recipe.get("repeat_until_stable", False)
+    max_passes = recipe.get("max_passes", DEFAULT_PASSES) if repeat else 1
+    deadline = time.monotonic() + TIME_BUDGET_SECONDS if deadline is None else deadline
+
+    work = {a.number: t for a, t in paragraphs}
+    original = dict(work)
+    address = {a.number: a for a, _ in paragraphs}
+    direct: dict[int, list[dict[str, Any]]] = {}  # pass-1 text edits, kept when no 2nd pass
+    others: list[dict[str, Any]] = []  # every other operation, from the original text only
+    seen: set[tuple[str, str]] = set()
+    redone: set[int] = set()  # paragraphs whose text changed in a pass after the first
+    deleted: set[int] = set()
+    warnings: list[str] = []
+    total = 0
+    passes = 0
+    stable = False
+    capped = False
+    to_scan = sorted(n for n, t in work.items() if t)
+
+    for passno in range(1, max_passes + 1):
+        passes = passno
+        found, stopped = _scan(options, matcher, [(n, work[n]) for n in to_scan], deadline)
+        if stopped is not None:
+            warnings.append(f"TIME_BUDGET: stopped at p{stopped}; the proposal is partial")
+            capped = True
+        changed: list[int] = []
+        for number, spans in found:
+            if number in deleted:
+                continue
+            text = work[number]
+            addr = address[number]
+            edits: list[tuple[int, int, str]] = []
+            for start, end, groups in spans:
+                if passno == 1:
+                    total += 1
+                matched = text[start:end]
+                for step in steps:
+                    kind = step["op"]
+                    if kind != "text.replace" and passno > 1:
+                        continue  # other operations are decided on the text as read
+                    op = _instantiate(step, addr, start, end, matched, text, groups)
+                    if kind == "text.replace":
+                        lo, hi = (0, len(text)) if step["at"] == "$para" else (start, end)
+                        if text[lo:hi] == op["text"]:
+                            continue  # already what the recipe wants
+                        edits.append((lo, hi, op["text"]))
+                        if passno == 1:
+                            direct.setdefault(number, []).append(op)
+                        continue
+                    if kind == "para.delete":
+                        deleted.add(number)
+                    if (kind, op["at"]) not in seen:
+                        seen.add((kind, op["at"]))
+                        others.append(op)
+            if number in deleted:
+                continue
+            new = text
+            for start, end, replacement in reversed(edits):
+                new = new[:start] + replacement + new[end:]
+            if new != text:
+                changed.append(number)
+                work[number] = new
+                if passno > 1:
+                    redone.add(number)
+        if len(others) + sum(map(len, direct.values())) > MAX_OPS:
+            warnings.append(f"OPS_CAP: more than {MAX_OPS} operations; narrow the recipe")
+            capped = True
+        if capped:
+            break
+        if not changed:
+            stable = True
+            break
+        to_scan = [n for n in changed if work[n]]
+    if not stable and not capped and repeat:
+        warnings.append(f"NOT_STABLE: still changing after {max_passes} passes")
+    if capped:
+        stable = False
+
+    ops: list[dict[str, Any]] = []
+    for number in sorted(set(direct) | redone):
+        if number in deleted:
+            continue
+        if number in redone:
+            start, end, new = _smallest_edit(original[number], work[number])
+            if (start, end, new) != (0, 0, ""):
+                ops.append(
+                    {"op": "text.replace", "at": f"{address[number]}:{start}-{end}", "text": new}
+                )
+        else:
+            ops.extend(direct[number])
+    ops.extend(others)
+    return Proposal(ops, passes, stable, total, warnings)
+
+
+def _instantiate(
+    step: dict[str, Any],
+    addr: Address,
+    start: int,
+    end: int,
+    matched: str,
+    para: str,
+    groups: Sequence[str],
+) -> dict[str, Any]:
+    out = {k: v for k, v in step.items() if k not in ("op", "at")}
+    if "text" in out:
+        out["text"] = _substitute(out["text"], matched, para, groups)
+    at = str(addr) if step["at"] == "$para" else f"{addr}:{start}-{end}"
+    return {"op": step["op"], "at": at, **out}

doc_control/recipes/schema.py

@@ -0,0 +1,140 @@
+"""The strict schema of a declarative recipe, and its validation.
+
+A recipe is pure data. Every object, at every level, rejects keys the schema does not
+list (``RECIPE_INVALID``), so a recipe has no way to name code, files or the network. The
+only things a recipe can ask for are the batch operations in :data:`PROPOSABLE`, aimed at
+``$match`` (the matched range) or ``$para`` (its paragraph). Text may use ``$match``,
+``$para`` and ``$1``..``$9`` (``$$`` is a literal dollar sign); nothing is evaluated.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from doc_control.errors import DocControlError, ErrorCode
+from doc_control.model.search import Matcher, SearchOptions
+from doc_control.ops.comments import COMMENT_SCHEMAS, comment_from_dict
+from doc_control.ops.tracked import OPERATION_SCHEMAS, edit_from_dict
+from doc_control.schema import validate
+
+RECIPE_VERSION = 1
+NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,63}")
+SUBSTITUTION = re.compile(r"\$(\$|match|para|[1-9])")
+PLACEHOLDER = "p1#00000000"  # stands in for `at` when checking an operation's other fields
+TARGETS = ("$match", "$para")
+PROPOSABLE = ("text.replace", "format.apply", "style.apply", "para.delete", "comment.add")
+MAX_PROPOSE = 8
+MAX_PASSES = 10
+DEFAULT_PASSES = 4
+
+_FIND: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "regex": {"type": "string", "minLength": 1},
+        "literal": {"type": "string", "minLength": 1},
+        "ignore_case": {"type": "boolean"},
+        "ignore_accents": {"type": "boolean"},
+        "whole_word": {"type": "boolean"},
+        "scope": {"enum": ["paragraph"]},
+    },
+    "additionalProperties": False,
+}
+
+RECIPE_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "recipe": {"type": "string", "minLength": 1},
+        "version": {"enum": [RECIPE_VERSION]},
+        "description": {"type": "string"},
+        "match": {
+            "type": "object",
+            "properties": {"find": _FIND},
+            "required": ["find"],
+            "additionalProperties": False,
+        },
+        "propose": {"type": "array", "items": {"type": "object"}},
+        "repeat_until_stable": {"type": "boolean"},
+        "max_passes": {"type": "integer", "minimum": 1, "maximum": MAX_PASSES},
+    },
+    "required": ["recipe", "version", "match", "propose"],
+    "additionalProperties": False,
+}
+
+
+def _invalid(message: str, **details: Any) -> DocControlError:
+    return DocControlError(ErrorCode.RECIPE_INVALID, message, **details)
+
+
+def _checked(value: Any, schema: dict[str, Any], path: str) -> None:
+    try:
+        validate(value, schema, path)
+    except DocControlError as err:
+        raise _invalid(err.message, **err.details) from err
+
+
+def search_options(recipe: dict[str, Any]) -> SearchOptions:
+    find = recipe["match"]["find"]
+    return SearchOptions(
+        find.get("regex", find.get("literal")),
+        regex="regex" in find,
+        ignore_case=find.get("ignore_case", False),
+        ignore_accents=find.get("ignore_accents", False),
+        whole_word=find.get("whole_word", False),
+    )
+
+
+def validate_recipe(raw: Any) -> dict[str, Any]:
+    """Return ``raw`` if it is a well-formed recipe, else raise ``RECIPE_INVALID``."""
+    _checked(raw, RECIPE_SCHEMA, "$")
+    if NAME.fullmatch(raw["recipe"]) is None:
+        raise _invalid("$.recipe: use lower-case letters, digits and hyphens (at most 64)")
+    find = raw["match"]["find"]
+    if ("regex" in find) == ("literal" in find):
+        raise _invalid("$.match.find: give exactly one of `regex` and `literal`")
+    options = search_options(raw)
+    try:
+        matcher = Matcher(options)
+    except DocControlError as err:
+        raise _invalid(f"$.match.find: {err.message}") from err
+    groups = matcher.pattern.groups if options.regex else 0
+    steps = raw["propose"]
+    if not 1 <= len(steps) <= MAX_PROPOSE:
+        raise _invalid(f"$.propose: give 1 to {MAX_PROPOSE} operations")
+    repeat = raw.get("repeat_until_stable", False)
+    if "max_passes" in raw and not repeat:
+        raise _invalid("$.max_passes: only meaningful with `repeat_until_stable`")
+    kinds = [
+        _check_step(step, f"$.propose[{i}]", groups, options, repeat)
+        for i, step in enumerate(steps)
+    ]
+    if kinds.count("text.replace") > 1:
+        raise _invalid("$.propose: at most one text.replace")
+    if "para.delete" in kinds and len(kinds) > 1:
+        raise _invalid("$.propose: para.delete cannot be combined with other operations")
+    return dict(raw)
+
+
+def _check_step(step: Any, path: str, groups: int, options: SearchOptions, repeat: bool) -> str:
+    if not isinstance(step, dict) or step.get("op") not in PROPOSABLE:
+        raise _invalid(f"{path}.op: must be one of {list(PROPOSABLE)}")
+    kind: str = step["op"]
+    _checked(step, OPERATION_SCHEMAS.get(kind) or COMMENT_SCHEMAS[kind], path)
+    at = step["at"]
+    if at not in TARGETS or (kind == "para.delete" and at != "$para"):
+        allowed = "$para" if kind == "para.delete" else "$match or $para"
+        raise _invalid(f"{path}.at: must be {allowed}")
+    if repeat and at == "$match" and kind != "text.replace":
+        raise _invalid(f"{path}.at: with `repeat_until_stable` only text.replace may target $match")
+    for ref in SUBSTITUTION.findall(step.get("text", "")):
+        if ref.isdigit() and int(ref) > groups:
+            raise _invalid(f"{path}.text: ${ref} but the pattern has {groups} group(s)")
+    probe = {**step, "at": PLACEHOLDER}
+    try:
+        if kind == "comment.add":
+            comment_from_dict(probe)
+        else:
+            edit_from_dict(probe)
+    except DocControlError as err:
+        raise _invalid(f"{path}: {err.message}") from err
+    return kind

doc_control/recipes/store.py

@@ -0,0 +1,96 @@
+"""Where recipes come from: bundled data files and the user store.
+
+User recipes are ``<user home>/recipes/<name>.json`` (``DOC_CONTROL_HOME`` or the
+platform default, see :func:`doc_control.safety.store.user_home`). Files are read as plain
+JSON and validated against the strict schema; a bad file never stops the others and is
+reported by :func:`catalogue`. A user recipe cannot take the name of a bundled one.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from doc_control.errors import DocControlError, ErrorCode
+from doc_control.recipes.schema import validate_recipe
+from doc_control.safety.store import user_home
+
+MAX_RECIPE_BYTES = 64 * 1024
+BUNDLED_DIR = Path(__file__).parent / "bundled"
+
+
+@dataclass(frozen=True)
+class Catalogue:
+    recipes: dict[str, tuple[str, dict[str, Any]]]  # name -> (source, recipe)
+    invalid: list[dict[str, str]]
+
+
+def user_recipes_dir(env: Mapping[str, str] | None = None) -> Path:
+    return user_home(env) / "recipes"
+
+
+def _no_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in out:
+            raise ValueError(f"duplicate key {key!r}")
+        out[key] = value
+    return out
+
+
+def _no_constant(name: str) -> Any:
+    raise ValueError(f"{name} is not JSON")
+
+
+def parse_recipe_file(path: Path) -> dict[str, Any]:
+    """Read and validate one recipe file; ``RECIPE_INVALID`` if it is not a valid recipe."""
+    try:
+        if path.stat().st_size > MAX_RECIPE_BYTES:
+            raise DocControlError(ErrorCode.RECIPE_INVALID, f"larger than {MAX_RECIPE_BYTES} bytes")
+        raw = json.loads(
+            path.read_text(encoding="utf-8"),
+            object_pairs_hook=_no_duplicates,
+            parse_constant=_no_constant,
+        )
+    except (OSError, ValueError) as exc:  # JSONDecodeError and UnicodeDecodeError are ValueErrors
+        raise DocControlError(ErrorCode.RECIPE_INVALID, f"cannot read recipe: {exc}") from exc
+    recipe = validate_recipe(raw)
+    if path.stem != recipe["recipe"]:
+        raise DocControlError(
+            ErrorCode.RECIPE_INVALID, f"file name must be {recipe['recipe']}.json"
+        )
+    return recipe
+
+
+def _load_dir(
+    directory: Path,
+    source: str,
+    found: dict[str, tuple[str, dict[str, Any]]],
+    bad: list[dict[str, str]],
+) -> None:
+    if not directory.is_dir():
+        return
+    for path in sorted(directory.glob("*.json")):
+        try:
+            if path.is_symlink():
+                raise DocControlError(ErrorCode.RECIPE_INVALID, "symbolic links are not followed")
+            recipe = parse_recipe_file(path)
+            if recipe["recipe"] in found:
+                raise DocControlError(
+                    ErrorCode.RECIPE_INVALID, f"the name {recipe['recipe']} is already taken"
+                )
+        except DocControlError as err:
+            bad.append({"file": path.name, "source": source, "error": err.message})
+            continue
+        found[recipe["recipe"]] = (source, recipe)
+
+
+def catalogue(env: Mapping[str, str] | None = None, bundled_dir: Path | None = None) -> Catalogue:
+    found: dict[str, tuple[str, dict[str, Any]]] = {}
+    bad: list[dict[str, str]] = []
+    _load_dir(BUNDLED_DIR if bundled_dir is None else bundled_dir, "bundled", found, bad)
+    _load_dir(user_recipes_dir(env), "user", found, bad)
+    return Catalogue(found, bad)

doc_control/registry.py

@@ -0,0 +1,76 @@
+"""Operation registry: one JSON Schema per operation, shared by CLI and MCP."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from doc_control.envelope import Reply
+from doc_control.schema import check_schema, validate
+
+# A handler returns its result dict, or a Reply to also fill `doc`, `warnings` and `next`.
+Handler = Callable[[dict[str, Any]], dict[str, Any] | Reply]
+
+
+@dataclass(frozen=True)
+class Operation:
+    name: str
+    kind: Literal["read", "write"]
+    summary: str
+    schema: dict[str, Any]
+    handler: Handler
+
+    def run(self, request: dict[str, Any]) -> dict[str, Any] | Reply:
+        validate(request, self.schema)
+        return self.handler(request)
+
+
+REGISTRY: dict[str, Operation] = {}
+
+
+def operation(
+    name: str, kind: Literal["read", "write"], summary: str, schema: dict[str, Any]
+) -> Callable[[Handler], Handler]:
+    """Register a handler; the schema must be a well-formed object schema."""
+
+    def wrap(handler: Handler) -> Handler:
+        if name in REGISTRY:
+            raise ValueError(f"operation registered twice: {name}")
+        if schema.get("type") != "object":
+            raise ValueError(f"schema for {name} must be an object schema")
+        problems = check_schema(schema)
+        if problems:
+            raise ValueError(f"schema for {name} is malformed: {'; '.join(problems)}")
+        REGISTRY[name] = Operation(name, kind, summary, schema, handler)
+        return handler
+
+    return wrap
+
+
+def load_operations() -> dict[str, Operation]:
+    """Import every operation module so their decorators run."""
+    from doc_control.ops import (  # noqa: F401
+        anchor,
+        batch,
+        comments,
+        compare,
+        diff,
+        dotx,
+        find,
+        history,
+        layout,
+        locate,
+        meta,
+        page,
+        read,
+        recipe,
+        rename,
+        revision,
+        segment,
+        styleset,
+        stylewrite,
+        text,
+    )
+
+    return REGISTRY

doc_control/safety/__init__.py

@@ -0,0 +1,38 @@
+"""Safety layer: checks and a backup that run before any write to a document."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from datetime import datetime
+from pathlib import Path
+
+from doc_control.errors import DocControlError, ErrorCode
+from doc_control.safety.lock import check_changed, check_unlocked
+from doc_control.safety.store import backup
+
+__all__ = ["guard_write"]
+
+
+def guard_write(
+    doc_path: Path,
+    expect_sha256: str | None,
+    *,
+    now: datetime | None = None,
+    env: Mapping[str, str] | None = None,
+) -> Path:
+    """Run before any write: lock check, change check, then a backup.
+
+    Raises ``DOC_NOT_FOUND``, ``DOC_LOCKED`` or ``DOC_CHANGED_ON_DISK`` without
+    touching the document or creating a store; returns the backup path. The
+    backup re-checks ``expect_sha256`` against the bytes it copied, so a change
+    landing between the check and the copy is ``DOC_CHANGED_ON_DISK`` too, with
+    no backup left behind. A bad store config is ``SCHEMA_INVALID`` and a
+    corrupt store id ``INTERNAL``, both raised before anything is copied.
+    """
+    if not doc_path.is_file():
+        raise DocControlError(
+            ErrorCode.DOC_NOT_FOUND, f"no such file: {doc_path}", path=str(doc_path)
+        )
+    check_unlocked(doc_path)
+    check_changed(doc_path, expect_sha256)
+    return backup(doc_path, now=now, env=env, expect_sha256=expect_sha256)