lockwire 0.2.0__py3-none-any.whl

lockwire/bench.py

@@ -0,0 +1,182 @@
+"""`lockwire bench`: score detection and forwarding separately, on a corpus.
+
+Two measurements, never merged into one number (detection and forwarding):
+
+1. **Detection** — for each generated out-of-bound plan, does the *existing*
+   evaluator (`plan.validate_plan`, reused — this module never
+   re-walks a plan) flag it? Categorized as ``detected`` / ``missed`` (a
+   violating case not flagged; "leaked" is a detection
+   metric only, and a correctly-forwarded violation is a detection, not a
+   leak) / ``false_alarm`` (a compliant case wrongly flagged) /
+   ``unscorable_correct`` / ``unscorable_mismatch``.
+2. **Forwarding** — every case reaches the simulated driver byte-identical,
+   in order, exactly once, *regardless of verdict* (lockwire forwards
+   the command unchanged, never originates one, never alters one, never
+   withholds one — lockwire has no gate to leak past; scoring forwarding on
+   the verdict would build the wrong reading).
+
+Self-proving: a corpus that scores well against an evaluator with
+the checks deleted is measuring itself, not the product. `run_bench` takes
+the evaluator as a parameter for exactly this reason — see
+`tests/test_bench.py::test_checks_deleted_detection_rate_collapses`.
+
+Ceiling, published in `format_report` itself, not only here: this corpus
+(`tests/fixtures/bench/corpus.yaml`) is hand-authored by the same project
+that wrote the checks it exercises. A bench that only scores cases it wrote
+to be catchable is marketing, not measurement.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Literal
+
+from lockwire.plan import Plan, PlanReport, Step, validate_plan
+from lockwire.report import Verdict, verdict
+from lockwire.schema import Contract, _load_yaml_guarded, _reject_yaml_anchors_aliases, _yaml
+
+DEFAULT_CORPUS_PATH = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "bench" / "corpus.yaml"
+
+Category = Literal["violating", "compliant", "unscorable"]
+Evaluator = Callable[[Contract, Plan], PlanReport]
+
+
+@dataclass(frozen=True, slots=True)
+class BenchCase:
+    id: str
+    category: Category
+    step: Step
+
+
+def load_corpus(path: Path) -> list[BenchCase]:
+    """Load the corpus YAML (id/category/step per case). Same guarded loader
+    as `plan.load_plan` and `schema.load_contract` — a corpus file is trusted
+    exactly as much as any other checked-in lockfile-shaped document."""
+    with open(path, encoding="utf-8") as f:
+        data = _load_yaml_guarded(_yaml(), f)
+    _reject_yaml_anchors_aliases(data)
+    cases = []
+    for raw in data["cases"]:
+        # Reuse the plan's own discriminated Step union to validate the case
+        # instead of a second step parser.
+        step = Plan.model_validate({"steps": [raw["step"]]}).steps[0]
+        cases.append(BenchCase(id=raw["id"], category=raw["category"], step=step))
+    return cases
+
+
+def _score(contract: Contract, step: Step, evaluator: Evaluator) -> Verdict:
+    return verdict(evaluator(contract, Plan(steps=[step])))
+
+
+def _classify(case: BenchCase, v: Verdict) -> str:
+    if case.category == "violating":
+        return "detected" if v.code == "NO_GO" else "missed"
+    if case.category == "compliant":
+        return "false_alarm" if v.code == "NO_GO" else "correct_pass"
+    return "unscorable_correct" if v.code == "NOT_SCORED" else "unscorable_mismatch"
+
+
+def score_and_forward(
+    contract: Contract, step: Step, evaluator: Evaluator = validate_plan
+) -> tuple[Verdict, Step]:
+    """Score one step and forward it, unaltered, to the simulated driver.
+
+    The forward always happens: lockwire has no gate, so nothing it
+    scores is ever withheld, including a step scored NO-GO or NOT SCORED.
+    Returns ``(verdict, the forwarded step)``; the caller appends the
+    forwarded step to its sink so byte-identity is checkable independently
+    of the verdict.
+    """
+    forwarded = step.model_copy(deep=True)
+    return _score(contract, step, evaluator), forwarded
+
+
+@dataclass(frozen=True, slots=True)
+class BenchResult:
+    corpus_size: int
+    detected: int
+    missed: int
+    false_alarms: int
+    correct_pass: int
+    unscorable_correct: int
+    unscorable_mismatch: int
+    forwarded: int
+
+    @property
+    def violating_total(self) -> int:
+        return self.detected + self.missed
+
+    @property
+    def detection_rate(self) -> float | None:
+        if self.violating_total == 0:
+            return None
+        return self.detected / self.violating_total
+
+    @property
+    def exit_code(self) -> int:
+        # The bench tool's own exit reflects whether a hole was caught, not
+        # the verdict words of the individual plans it scored (accept line).
+        return 1 if self.missed else 0
+
+
+_BUCKETS = (
+    "detected",
+    "missed",
+    "false_alarm",
+    "correct_pass",
+    "unscorable_correct",
+    "unscorable_mismatch",
+)
+
+
+def run_bench(
+    contract: Contract, cases: list[BenchCase], evaluator: Evaluator = validate_plan
+) -> tuple[BenchResult, list[Step]]:
+    """Run every case: score it (detection) and forward it (always). Returns
+    the tallied result plus the simulated driver's received-step sink, in
+    the order the corpus was walked."""
+    counts = dict.fromkeys(_BUCKETS, 0)
+    sink: list[Step] = []
+    for case in cases:
+        v, forwarded = score_and_forward(contract, case.step, evaluator)
+        sink.append(forwarded)
+        counts[_classify(case, v)] += 1
+    result = BenchResult(
+        corpus_size=len(cases),
+        detected=counts["detected"],
+        missed=counts["missed"],
+        false_alarms=counts["false_alarm"],
+        correct_pass=counts["correct_pass"],
+        unscorable_correct=counts["unscorable_correct"],
+        unscorable_mismatch=counts["unscorable_mismatch"],
+        forwarded=len(sink),
+    )
+    return result, sink
+
+
+def format_report(result: BenchResult, corpus_path: Path) -> str:
+    """The published detection rate, plain words — what the corpus is, how
+    many cases, what a catch counts as, and the self-scoring ceiling, all in
+    the output itself (not only in a docstring)."""
+    rate = result.detection_rate
+    rate_str = "n/a (no violating cases in the corpus)" if rate is None else f"{rate:.0%}"
+    return "\n".join(
+        [
+            f"corpus: {corpus_path} — {result.corpus_size} hand-authored cases",
+            f"detection rate: {rate_str} "
+            f"({result.detected}/{result.violating_total} violating cases flagged NO-GO; "
+            'a "catch" = the existing plan evaluator scores the case NO_GO)',
+            f"missed (leaked — a violating case the evaluator did not flag): {result.missed}",
+            f"false alarms (a compliant case wrongly flagged NO-GO): {result.false_alarms}",
+            f"unscorable cases correctly NOT SCORED: {result.unscorable_correct}"
+            + (
+                f" ({result.unscorable_mismatch} scored when they should not have been)"
+                if result.unscorable_mismatch
+                else ""
+            ),
+            f"forwarded to the simulated driver: {result.forwarded}/{result.corpus_size}, "
+            "unchanged, in order, once each, regardless of verdict (never withheld)",
+            "ceiling: this corpus is hand-authored by the same project that wrote the checks "
+            "it exercises; a bench that scores only cases it authored to be catchable is "
+            "marketing, not an independent measurement.",
+        ]
+    )

lockwire/blame.py

@@ -0,0 +1,100 @@
+"""One provenance row per declared record, optionally attributed by git."""
+
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from ruamel.yaml import YAML
+
+from lockwire import gitref
+from lockwire.schema import load_contract
+
+
+def _path_part(item: Any, index: int) -> str:
+    if isinstance(item, dict):
+        for key in ("id", "name"):
+            if key in item:
+                return str(item[key])
+    return str(index)
+
+
+def _plain(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {str(key): _plain(child) for key, child in value.items()}
+    if isinstance(value, list):
+        return [_plain(child) for child in value]
+    if isinstance(value, (dt.date, dt.datetime)):
+        return value.isoformat()
+    return value
+
+
+def _declared(node: Any, path: tuple[str, ...] = ()) -> Iterator[dict[str, Any]]:
+    if isinstance(node, dict):
+        provenance = node.get("provenance")
+        if isinstance(provenance, dict):
+            lines = {node.lc.key("provenance")[0] + 1}
+            lines.update(provenance.lc.key(key)[0] + 1 for key in provenance)
+            row = {
+                "path": "/".join(path),
+                "value": _plain({key: value for key, value in node.items() if key != "provenance"}),
+                "witness": str(provenance["witness"]),
+                "source": str(provenance["source"]),
+                "locked_at": str(provenance["locked_at"]),
+                "lines": lines,
+            }
+            if provenance.get("compared_from") is not None:
+                row["compared_from"] = str(provenance["compared_from"])
+            yield row
+        for key, child in node.items():
+            if key != "provenance":
+                yield from _declared(child, (*path, str(key)))
+    elif isinstance(node, list):
+        for index, child in enumerate(node):
+            yield from _declared(child, (*path, _path_part(child, index)))
+
+
+def _commits_by_line(porcelain: str) -> dict[int, str]:
+    """Per-line commit shas from a ``git blame --porcelain`` transcript."""
+    commits: dict[int, str] = {}
+    for line in porcelain.splitlines():
+        fields = line.split()
+        if (
+            len(fields) >= 3
+            and len(fields[0]) == 40
+            and all(c in "0123456789abcdef" for c in fields[0])
+        ):
+            commits[int(fields[2])] = fields[0]
+    return commits
+
+
+def rows(contract: Path) -> tuple[list[dict[str, Any]], bool]:
+    """Validated provenance rows and whether the path belongs to a repository.
+
+    ``last_changing_commit`` is omitted when git has no honest attribution (for
+    example an untracked file). The stored witness is rendered verbatim: a
+    persisted ``" (default)"`` marker is evidence; machine configuration is not.
+    """
+    load_contract(contract)
+    document = YAML(typ="rt").load(contract)
+    result = list(_declared(document))
+
+    repo = gitref.repo_root(contract)
+    relpath = gitref.relpath_in_repo(contract, repo) if repo is not None else None
+    porcelain = gitref.blame(relpath, repo=repo) if relpath and repo else None
+    commits = _commits_by_line(porcelain or "")
+    history = (
+        gitref.log(None, [relpath], repo=repo, follow=True)
+        if porcelain and relpath and repo
+        else None
+    )
+    order = {sha: index for index, (sha, _, _, _) in enumerate(history or [])}
+    for row in result:
+        candidates = {commits[line] for line in row.pop("lines") if line in commits}
+        candidates.discard("0" * 40)
+        attributed = [(order[sha], sha) for sha in candidates if sha in order]
+        if attributed:
+            row["last_changing_commit"] = min(attributed)[1]
+    return result, repo is not None

lockwire/catalog.py

@@ -0,0 +1,100 @@
+"""The parts catalog — a library of parts a user might pick from.
+
+Deliberately NOT part of the contract schema (`schema.py`, frozen). A contract is a
+declared, witnessed statement about the robot you have; a catalog is a library of parts you might
+use to build one. They answer different questions and don't share a lifecycle — a catalog entry
+changes when a vendor revises a datasheet, a contract entry changes when this robot's build
+changes. Keeping the catalog its own model in its own file means the freeze on `schema.py` is
+never touched: no field here is a contract field, so there is nothing to validate against
+the contract schema.
+
+Every number on a catalog part carries a `kind` and a `source`: a real
+number quotes where it came from; a placeholder says plainly that it is a placeholder. Nothing on
+a catalog part is presented as a real number without one of these two badges — an unlabelled
+number is a bug in the seed data, not a permitted state (see `_source_not_blank` below and
+`tests/test_catalog.py::test_every_seeded_number_has_a_kind_and_a_source`).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from ruamel.yaml import YAML
+
+_SEED_PATH = Path(__file__).parent / "catalog_seed.yaml"
+
+# "datasheet": a manufacturer datasheet, spec page or product-listing figure, cited by URL/name.
+# "source_file": a real number read out of a file already in this repo (e.g. a vendor URDF's
+#   <mass> tag) — real, but the citation is a file, not a published datasheet.
+# "assumed": fabricated for seeding, no real part backs it; the source string must say so.
+NumberKind = Literal["datasheet", "source_file", "assumed"]
+
+
+class SourcedNumber(BaseModel):
+    """One number on a catalog part, badged with where it came from."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    value: float
+    unit: str
+    kind: NumberKind
+    source: str
+
+    @field_validator("source")
+    @classmethod
+    def _source_not_blank(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("source must not be blank — a number without a source is not allowed")
+        return v
+
+
+class CatalogPart(BaseModel):
+    """One entry in the catalog: a part a user might pick, not a claim about a robot they have.
+
+    Reconciliation with `schema.py`'s `DeviceModel`: two vocabularies for "a part"
+    now exist and they are deliberately not one. `CatalogPart` is the SHOPPING vocabulary — a part
+    you might pick, sourced numbers, no robot attached. `DeviceModel` is the REGISTRY vocabulary —
+    checks, hard limits and known failure modes for a device on a robot you actually have. They
+    share only `id` and `family`; nothing else lines up because the two answer different
+    questions and don't share a lifecycle (see the module docstring). A consumer that needs both
+    must go through one door (the one-door rule), not read both models directly — `grep -rln
+    CatalogPart src/lockwire/ tests/` names every module that may reference this class, and
+    `test_no_module_straddles_catalog_and_registry_vocabularies` below fails the day that set
+    grows to include a module that also references `DeviceModel`. Merging the two, if ever wanted,
+    is a `schema.py` change and therefore a maintainer's call — not a refactor to do
+    on a reading of "decide the reconciliation."
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    family: str
+    name: str
+    numbers: dict[str, SourcedNumber] = Field(default_factory=dict)
+    note: str | None = None
+
+
+class Catalog(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    parts: list[CatalogPart]
+
+    @field_validator("parts")
+    @classmethod
+    def _ids_unique(cls, v: list[CatalogPart]) -> list[CatalogPart]:
+        seen: set[str] = set()
+        for part in v:
+            if part.id in seen:
+                raise ValueError(f"duplicate catalog part id: {part.id}")
+            seen.add(part.id)
+        return v
+
+
+def load_catalog(path: Path | str | None = None) -> Catalog:
+    """Read the catalog from `path` (default: the seeded catalog shipped with lockwire)."""
+    yaml = YAML(typ="safe")
+    p = Path(path) if path is not None else _SEED_PATH
+    data = yaml.load(p.read_text())
+    return Catalog.model_validate(data)