booley-rtl 0.1.0__py3-none-any.whl

booley/__init__.py

@@ -0,0 +1,13 @@
+"""Booley — orchestrated RTL development harness."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    # Distribution name on PyPI is "booley-rtl"; the import package is "booley".
+    __version__ = version("booley-rtl")
+except PackageNotFoundError:
+    try:
+        # Pre-rename dev/editable installs registered the old distribution name.
+        __version__ = version("booley")
+    except PackageNotFoundError:
+        __version__ = "0.0.0-dev"

booley/adapterlib/__init__.py

@@ -0,0 +1,89 @@
+"""Public helper library for Project-native B-Tool adapters.
+
+This package is the ONE sanctioned Booley import for project adapters
+(``.booley_project/adapters/*.py``). Everything else under ``booley.*`` stays
+private to adapters: the JSON request/response files remain the only contract
+(docs/BTOOL-ADAPTER-CONTRACTS.md), and this library only makes satisfying that
+contract mechanical. Adapters work without it; with it, the recurring blocks —
+request parsing, response envelopes, identity echoes, stale-artifact guards,
+budgeted subprocess runs, finding/metric normalization, and pre-write
+self-validation against Booley's own validator — come from tested code instead
+of being re-authored per project.
+
+Import surface is guaranteed available wherever adapters run: Booley invokes
+adapters with its own interpreter (``sys.executable``), both in the sandbox
+and on the host, so the installed ``booley`` package is importable by
+construction and can never version-skew against the invoking side.
+
+Typical adapter::
+
+    from booley import adapterlib as al
+
+    def build(req: al.AdapterRequest) -> dict:
+        command = ["make", "sim", f"CONFIG={req.target_name}", f"TEST={req.test}"]
+        if req.dry_run:
+            return al.dry_run_response(req, " ".join(command))
+        report = al.fresh_artifact(req.work_dir / "build" / "run_case.report")
+        run = al.run_command(command, req)
+        verdict = al.derive_sim_verdict(returncode=run.returncode,
+                                        log_text=run.output, timed_out=run.timed_out)
+        return al.response(req, verdict, command=run.command,
+                           log_path=run.log_path, elapsed_s=run.elapsed_s)
+
+    if __name__ == "__main__":
+        raise SystemExit(al.main(build))
+"""
+
+from .parsers import (
+    SIM_RESULT_FAILED,
+    SIM_RESULT_PASSED,
+    VERILATOR_ERROR_RE,
+    VERILATOR_WARNING_RE,
+    area_to_kge,
+    count_sva_errors,
+    derive_sim_verdict,
+    extract_error_gist,
+    first_error_line,
+    parse_area_from_stat,
+    parse_sim_verdict,
+    parse_verilator_findings,
+)
+from .proc import CommandResult, fresh_artifact, run_command
+from .request import AdapterRequest
+from .responses import (
+    AdapterError,
+    clock_timing,
+    dry_run_response,
+    finding,
+    normalize_severity,
+    response,
+    synthesis_metrics,
+)
+from .runner import main
+
+__all__ = [
+    "SIM_RESULT_FAILED",
+    "SIM_RESULT_PASSED",
+    "VERILATOR_ERROR_RE",
+    "VERILATOR_WARNING_RE",
+    "AdapterError",
+    "AdapterRequest",
+    "CommandResult",
+    "area_to_kge",
+    "clock_timing",
+    "count_sva_errors",
+    "derive_sim_verdict",
+    "dry_run_response",
+    "extract_error_gist",
+    "finding",
+    "first_error_line",
+    "fresh_artifact",
+    "main",
+    "normalize_severity",
+    "parse_area_from_stat",
+    "parse_sim_verdict",
+    "parse_verilator_findings",
+    "response",
+    "run_command",
+    "synthesis_metrics",
+]

booley/adapterlib/parsers.py

@@ -0,0 +1,169 @@
+"""Log/report parsers shared by the built-in flows and Project-native adapters.
+
+WHAT: The pure text-extraction blocks of the built-in tools, importable by
+adapters: sim sentinel verdicts (re-exported from ``booley.sim.sim_result``),
+Verilator warning/error parsing (single source of truth — ``booley.tools.lint``
+imports these regexes), compiler-error gists (shared with
+``booley.tools.elaborate``), and Yosys area extraction (re-exported from
+``booley.yosys.syn_parse``).
+
+WHY: An adapter wrapping a Verilator/Icarus/Yosys-based project flow should
+not re-derive parsing the built-in flows already hardened (full-scan sentinel
+rule, QA-7 %Error-means-not-clean, location-less-finding dropping).
+
+CONSUMERS: project adapters via ``booley.adapterlib``; ``booley.tools.lint``
+(warning/error regexes) and ``booley.tools.elaborate`` (error gist).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# Re-exports: sentinel-based sim verdict + SVA counting (leaf module).
+from booley.sim.sim_result import (  # noqa: F401 — public adapterlib surface
+    SIM_RESULT_FAILED,
+    SIM_RESULT_PASSED,
+    count_sva_errors,
+    parse_sim_verdict,
+)
+
+# Re-exports: Yosys stat-file area extraction + kGE conversion (leaf module).
+from booley.yosys.syn_parse import (  # noqa: F401 — public adapterlib surface
+    area_to_kge,
+    parse_area_from_stat,
+)
+
+from .responses import finding
+
+# Verilator warning lines (also the built-in lint parser's regex):
+#   %Warning-UNUSEDSIGNAL: module_a.sv:42:5: Signal is not used: 'foo'
+VERILATOR_WARNING_RE = re.compile(
+    r"%Warning-(?P<rule>[A-Z0-9_]+):\s+"
+    r"(?P<file>[^:]+):(?P<line>\d+):(?P<col>\d+):\s+"
+    r"(?P<message>.+)"
+)
+
+# Verilator emits ``%Error`` (and the summary ``%Error: Exiting due to N
+# error(s)``) for hard failures — undeclared signals, parse errors — and exits
+# non-zero. A warnings-only parser yields zero findings on such a run and
+# would score it as a clean PASS (QA-7); scan for errors separately.
+VERILATOR_ERROR_RE = re.compile(r"^%Error.*", re.MULTILINE)
+
+# Located %Error lines — the ones that may become findings. The rule suffix is
+# optional (``%Error-BLKANDNBLK: ...`` vs plain ``%Error: file:line:col: ...``);
+# summary/epilogue errors carry no location and must NOT become findings.
+_VERILATOR_LOCATED_ERROR_RE = re.compile(
+    r"%Error(?:-(?P<rule>[A-Z0-9_]+))?:\s+"
+    r"(?P<file>[^\s:][^:]*):(?P<line>\d+):(?:(?P<col>\d+):)?\s+"
+    r"(?P<message>.+)"
+)
+
+
+def first_error_line(output: str) -> str | None:
+    """Return the first Verilator ``%Error`` line in *output*, if any."""
+    match = VERILATOR_ERROR_RE.search(output)
+    return match.group(0).strip() if match else None
+
+
+def parse_verilator_findings(output: str) -> list[dict[str, Any]]:
+    """Parse Verilator output into contract-shaped lint findings.
+
+    ``%Warning-RULE: file:line:col: msg`` → ``warning`` findings;
+    ``%Error[-RULE]: file:line[:col]: msg`` → ``error`` findings. Location-less
+    error/summary lines (``%Error: Exiting due to N error(s)``) are dropped —
+    Booley rejects findings without a ``file``. Presence of any ``%Error`` line
+    should still flip the verdict to ``fail``/``tool_error``; check
+    :func:`first_error_line` for that, independently of the findings list.
+    """
+    findings: list[dict[str, Any]] = []
+    for match in VERILATOR_WARNING_RE.finditer(output):
+        findings.append(finding(
+            rule=match.group("rule"),
+            severity="warning",
+            file=match.group("file"),
+            line=int(match.group("line")),
+            col=int(match.group("col")),
+            message=match.group("message"),
+        ))
+    for match in _VERILATOR_LOCATED_ERROR_RE.finditer(output):
+        col = match.group("col")
+        findings.append(finding(
+            rule=match.group("rule") or "ERROR",
+            severity="error",
+            file=match.group("file"),
+            line=int(match.group("line")),
+            col=int(col) if col else None,
+            message=match.group("message"),
+        ))
+    return findings
+
+
+def extract_error_gist(error_output: str) -> str:
+    """Extract a one-line error gist from compiler output.
+
+    Looks for common error patterns (Verilator, Icarus, sv2v) and returns the
+    first meaningful error line, truncated for display — a ready-made
+    ``first_error`` for simulate responses. (Also the built-in elaborate
+    tool's extractor.)
+    """
+    if not error_output:
+        return ""
+    error_re = re.compile(
+        r"(?:%Error[^:]*:\s*\S+:\d+:\s*(.+)"        # Verilator
+        r"|(?:^|\n)\s*(?:error|Error):\s*(.+)"      # generic
+        r"|(?:^|\n)\s*(\S+:\d+:\s*(?:error|syntax error).+))",  # Icarus/sv2v
+    )
+    m = error_re.search(error_output)
+    if m:
+        msg = next(g for g in m.groups() if g)
+        return msg.strip()[:80]
+    # Fallback: last non-empty line.
+    for line in reversed(error_output.splitlines()):
+        stripped = line.strip()
+        if stripped and not stripped.startswith(("-", "=")):
+            return stripped[:80]
+    return ""
+
+
+def derive_sim_verdict(
+    *,
+    returncode: int | None,
+    log_text: str,
+    timed_out: bool = False,
+    pass_sentinels: list[str] | None = None,
+    fail_sentinels: list[str] | None = None,
+) -> str:
+    """Derive a simulate verdict — exit code first, sentinels second.
+
+    Encodes the contract's "Verdict integrity" ordering for the common
+    self-checking-testbench case:
+
+    - ``timed_out`` → ``timeout``;
+    - any FAIL sentinel in the log → ``fail`` (a TB that prints FAIL failed,
+      whatever the exit code);
+    - non-zero exit with no fail sentinel → ``elab_error`` (the build/elab
+      died before the test could judge itself — never a functional ``fail``,
+      never a ``pass`` inherited from a stale artifact);
+    - clean exit + PASS sentinel → ``pass``;
+    - clean exit, no sentinel → ``inconclusive`` (a silent run is not
+      evidence).
+
+    Sentinels default to the built-in ``[SIM_RESULT] PASSED/FAILED`` markers;
+    pass the project's own wording (booley.toml ``[tools.simulate]``
+    pass_sentinels/fail_sentinels) to keep the testbench untouched. Flows with
+    richer semantics (SVA counting, per-test reports) should derive their own
+    verdict from the lower-level pieces instead.
+    """
+    if timed_out:
+        return "timeout"
+    sentinel = parse_sim_verdict(
+        log_text, pass_sentinels=pass_sentinels, fail_sentinels=fail_sentinels
+    )
+    if sentinel is False:
+        return "fail"
+    if returncode != 0:
+        return "elab_error"
+    if sentinel is True:
+        return "pass"
+    return "inconclusive"

booley/adapterlib/proc.py

@@ -0,0 +1,181 @@
+"""Command execution for Project-native adapters: log capture, timeout, freshness.
+
+WHAT: ``run_command`` runs one project flow command with its combined output
+streamed straight to a log file (never buffered whole in adapter memory), a
+timeout derived from the request budget, and process-tree kill on expiry.
+``fresh_artifact`` deletes result files a build/run rewrites, so a verdict can
+never be inherited from a previous run's artifact.
+
+WHY: These two blocks are where hand-rolled adapters produced false PASSes
+(contract, "Verdict integrity: never trust a stale artifact") and unstructured
+timeouts (Booley's hard kill leaves no evidence; a budget-aware command timeout
+returns a structured ``timeout`` verdict with a log instead).
+
+CONSUMERS: project adapters via ``booley.adapterlib``.
+"""
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from booley.platform_paths import kill_process_tree, popen_new_group_kwargs
+
+from .request import AdapterRequest
+
+# How much of the log tail to retain in CommandResult.output (and echo on
+# request) — enough for sentinel scans and error extraction without holding a
+# multi-GB simulation log in memory.
+_KEEP_OUTPUT_BYTES = 2_000_000
+
+
+@dataclass(frozen=True)
+class CommandResult:
+    """Outcome of one ``run_command`` invocation."""
+
+    command: str
+    returncode: int | None
+    timed_out: bool
+    elapsed_s: float
+    log_path: Path
+    # Tail of the combined stdout+stderr (capped; full text is in log_path).
+    output: str
+
+    @property
+    def ok(self) -> bool:
+        """True when the command exited 0 within its budget."""
+        return self.returncode == 0 and not self.timed_out
+
+    def tail(self, lines: int = 50) -> str:
+        """Last *lines* lines of the captured output — for ``error_tail`` fields."""
+        return "\n".join(self.output.splitlines()[-lines:])
+
+
+def run_command(
+    command: list[str] | str,
+    req: AdapterRequest | None = None,
+    *,
+    log_path: str | Path | None = None,
+    cwd: str | Path | None = None,
+    env: dict[str, str] | None = None,
+    timeout_s: float | None = None,
+    echo_tail: bool = True,
+) -> CommandResult:
+    """Run one project flow command with contract-friendly defaults.
+
+    - *command*: an argv list (exec'd directly), or a string (run through the
+      shell — needed for Makefiles that assume bash, cf. contract "Common
+      Pitfalls").
+    - *log_path*: combined stdout+stderr destination. Defaults to the request's
+      ``expected_log_path`` (where Booley looks for evidence), else
+      ``<artifact_dir>/<tool>.log``.
+    - *cwd*: defaults to the request's ``work_dir``.
+    - *env*: overlaid on the process environment after the request's ``env``.
+    - *timeout_s*: defaults to the remaining request budget minus a margin
+      (``req.remaining_s()``), so the adapter still has time to parse results
+      and write a structured ``timeout`` response before Booley's hard kill.
+      On expiry the whole process tree is killed.
+    - *echo_tail*: reprint the log tail to stdout afterwards, so the
+      ``adapter.stdout.log`` Booley archives shows what happened.
+    """
+    if log_path is None:
+        if req is None:
+            raise ValueError("run_command needs log_path when no request is given")
+        log_path = req.expected_log_path or (req.artifact_dir / f"{req.tool}.log")
+    log = Path(log_path)
+    log.parent.mkdir(parents=True, exist_ok=True)
+
+    shell = isinstance(command, str)
+    command_str = command if shell else shlex.join(command)
+    if timeout_s is None and req is not None:
+        timeout_s = req.remaining_s()
+
+    run_env = _merged_env(req, env)
+    start = time.monotonic()
+    # Stream output straight to the log file: no pipe to deadlock on, no
+    # multi-GB simulation log held in adapter memory.
+    with log.open("w", encoding="utf-8") as sink:
+        proc = subprocess.Popen(
+            command,
+            shell=shell,
+            cwd=str(cwd) if cwd is not None else (str(req.work_dir) if req else None),
+            stdout=sink,
+            stderr=subprocess.STDOUT,
+            env=run_env,
+            **popen_new_group_kwargs(),
+        )
+        timed_out = _wait(proc, timeout_s)
+    elapsed = time.monotonic() - start
+
+    output = _read_tail(log)
+    if timed_out:
+        output += f"\n[adapterlib] command timed out after {timeout_s:.0f}s: {command_str}\n"
+    if echo_tail:
+        print(f"[adapterlib] $ {command_str}\n{output}", flush=True)
+    return CommandResult(
+        command=command_str,
+        returncode=proc.returncode,
+        timed_out=timed_out,
+        elapsed_s=elapsed,
+        log_path=log,
+        output=output,
+    )
+
+
+def fresh_artifact(*paths: str | Path) -> Path | tuple[Path, ...]:
+    """Delete result files the upcoming build/run will (re)write.
+
+    Call BEFORE launching the flow for every artifact the verdict is read
+    from (a ``run_case.report``, a JUnit XML, a log sentinel file): anything
+    present afterwards then belongs to *this* invocation only, so a build that
+    fails to compile can never inherit the previous run's PASS (contract,
+    "Verdict integrity"). Returns the Path(s) for the post-run read-back.
+    """
+    cleared = tuple(Path(p) for p in paths)
+    for path in cleared:
+        path.unlink(missing_ok=True)
+    return cleared[0] if len(cleared) == 1 else cleared
+
+
+def _merged_env(req: AdapterRequest | None, env: dict[str, str] | None) -> dict[str, str] | None:
+    """Process env + request env + explicit overrides (None = inherit as-is)."""
+    if req is None and env is None:
+        return None
+    import os
+
+    merged = os.environ.copy()
+    if req is not None:
+        merged.update(req.env)
+    if env is not None:
+        merged.update({str(k): str(v) for k, v in env.items()})
+    return merged
+
+
+def _wait(proc: subprocess.Popen[Any], timeout_s: float | None) -> bool:
+    try:
+        proc.wait(timeout=timeout_s)
+    except subprocess.TimeoutExpired:
+        kill_process_tree(proc)
+        proc.wait()
+        return True
+    except KeyboardInterrupt:
+        kill_process_tree(proc)
+        raise
+    return False
+
+
+def _read_tail(log: Path, keep_bytes: int = _KEEP_OUTPUT_BYTES) -> str:
+    try:
+        size = log.stat().st_size
+        with log.open("rb") as handle:
+            if size > keep_bytes:
+                handle.seek(size - keep_bytes)
+            return handle.read().decode("utf-8", errors="replace")
+    except OSError as exc:  # log unreadable — surface why instead of crashing
+        print(f"[adapterlib] could not read log tail {log}: {exc}", file=sys.stderr)
+        return ""

booley/adapterlib/request.py

@@ -0,0 +1,184 @@
+"""Typed read-only view of a Project-native adapter request.
+
+WHAT: Wraps the ``request.json`` a B-Tool writes for a Project-native adapter
+(see docs/BTOOL-ADAPTER-CONTRACTS.md "Common Request") behind named accessors,
+so adapter code reads ``req.target_name`` instead of chasing raw dict keys —
+and gets the renamed-key mistakes (``config`` vs ``target``) for free.
+
+WHY: Every adapter used to hand-parse the request dict; typos in nested keys
+surfaced only as downstream ``contract_error``. A single typed view makes the
+request shape discoverable and keeps adapters resilient to additive schema
+growth (unknown keys are simply not exposed).
+
+CONSUMERS: project adapters via ``booley.adapterlib``; ``runner.main`` builds
+one per invocation.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# The tool-specific request section is not always named after the tool:
+# asic_synthesize nests its extras under "synthesis" (contract, "Common Request").
+_TOOL_SECTION_KEY = {
+    "simulate": "simulate",
+    "lint": "lint",
+    "asic_synthesize": "synthesis",
+}
+
+
+@dataclass(frozen=True)
+class AdapterRequest:
+    """One adapter invocation's request, plus where the response must go."""
+
+    raw: dict[str, Any]
+    request_path: Path
+    response_path: Path
+    # Monotonic clock at load time — anchors ``remaining_s`` so command
+    # timeouts leave room to write the response before Booley's hard kill.
+    loaded_at: float = field(default_factory=time.monotonic)
+
+    @classmethod
+    def load(cls, request_path: str | Path, response_path: str | Path) -> AdapterRequest:
+        """Read ``request.json`` and wrap it (raises on missing/invalid JSON)."""
+        path = Path(request_path)
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(raw, dict):
+            raise ValueError(f"adapter request must be a JSON object: {path}")
+        return cls(raw=raw, request_path=path, response_path=Path(response_path))
+
+    # -- common envelope ----------------------------------------------------
+
+    @property
+    def schema_version(self) -> int:
+        return int(self.raw.get("schema_version", 1))
+
+    @property
+    def tool(self) -> str:
+        return str(self.raw.get("tool", ""))
+
+    @property
+    def work_dir(self) -> Path:
+        return Path(str(self.raw.get("work_dir", ".")))
+
+    @property
+    def artifact_dir(self) -> Path:
+        return Path(str(self.raw.get("artifact_dir", ".")))
+
+    @property
+    def timeout_s(self) -> float:
+        try:
+            return float(self.raw.get("timeout_s", 600))
+        except (TypeError, ValueError):
+            return 600.0
+
+    @property
+    def dry_run(self) -> bool:
+        return bool(self.raw.get("dry_run", False))
+
+    @property
+    def env(self) -> dict[str, str]:
+        """Explicit environment additions supplied by Booley."""
+        raw = self.raw.get("env")
+        return {str(k): str(v) for k, v in raw.items()} if isinstance(raw, dict) else {}
+
+    # -- resolved Target ----------------------------------------------------
+
+    @property
+    def target(self) -> dict[str, Any]:
+        """The Booley-resolved Target object (``name``, ``defines``, ...)."""
+        raw = self.raw.get("target")
+        return dict(raw) if isinstance(raw, dict) else {}
+
+    @property
+    def target_name(self) -> str:
+        """The requested config name — echo back verbatim as ``resolved_target``."""
+        return str(self.target.get("name", ""))
+
+    @property
+    def top_module(self) -> str:
+        return str(self.raw.get("top_module", self.target.get("top_module", "")))
+
+    @property
+    def tb_top(self) -> str:
+        return str(self.raw.get("tb_top", self.target.get("tb_top", "")))
+
+    @property
+    def defines(self) -> list[str]:
+        raw = self.raw.get("defines", self.target.get("defines", []))
+        return [str(d) for d in raw] if isinstance(raw, list) else []
+
+    @property
+    def parameters(self) -> dict[str, Any]:
+        raw = self.raw.get("parameters", self.target.get("parameters", {}))
+        return dict(raw) if isinstance(raw, dict) else {}
+
+    @property
+    def tests(self) -> list[str]:
+        raw = self.raw.get("tests", self.target.get("tests", []))
+        return [str(t) for t in raw] if isinstance(raw, list) else []
+
+    # -- tool-specific section ----------------------------------------------
+
+    @property
+    def tool_section(self) -> dict[str, Any]:
+        """The tool-specific request block (``simulate``/``lint``/``synthesis``)."""
+        key = _TOOL_SECTION_KEY.get(self.tool, self.tool)
+        raw = self.raw.get(key)
+        return dict(raw) if isinstance(raw, dict) else {}
+
+    @property
+    def test(self) -> str | None:
+        """simulate: the resolved test name, or ``None`` when no test was selected.
+
+        When ``None``, the response must omit ``resolved_test`` (or return it as
+        null) — the contract forbids inventing one. ``responses.response`` obeys
+        this automatically.
+        """
+        value = self.tool_section.get("test")
+        return None if value is None else str(value)
+
+    @property
+    def trace(self) -> bool:
+        """simulate: whether a waveform trace was requested."""
+        return bool(self.tool_section.get("trace", False))
+
+    @property
+    def scope(self) -> str:
+        """lint: comma-separated file filter from ``--scope`` (empty = all files)."""
+        return str(self.tool_section.get("scope", ""))
+
+    @property
+    def expected_log_path(self) -> Path | None:
+        """Where Booley expects the tool log; a good default for ``run_command``."""
+        value = self.tool_section.get("expected_log_path")
+        return Path(str(value)) if value else None
+
+    @property
+    def expected_trace_path(self) -> Path | None:
+        """simulate: where Booley expects the trace when ``trace`` is true."""
+        value = self.tool_section.get("expected_trace_path")
+        return Path(str(value)) if value else None
+
+    @property
+    def expected_report_dir(self) -> Path | None:
+        """asic_synthesize: where Booley expects report files."""
+        value = self.tool_section.get("expected_report_dir")
+        return Path(str(value)) if value else None
+
+    # -- budget ---------------------------------------------------------------
+
+    def remaining_s(self, *, margin_s: float = 10.0, floor_s: float = 5.0) -> float:
+        """Seconds left of the request budget, minus *margin_s* to finish up.
+
+        Booley hard-kills the adapter process tree at ``timeout_s``; commands
+        launched with this budget leave the adapter time to parse results and
+        write the response, so a slow tool becomes a structured ``timeout``
+        verdict instead of a synthetic one with no evidence.
+        """
+        elapsed = time.monotonic() - self.loaded_at
+        return max(floor_s, self.timeout_s - elapsed - margin_s)