pi-dev-team 0.2.0

package/scripts/lib/session_report_shared.py

@@ -0,0 +1,262 @@
+"""session_report_shared.py — building blocks common to both
+session_report.py profiles (maintainer and downstream).
+
+Split out of session_report.py (issue #2098) so the unified CLI has clear
+domain layering: this module holds only what BOTH profiles need — the
+session_log vocabulary aliases, schema constants, plugin-version resolution,
+correction-cause classification, and skill/agent registry loading —
+so session_report_maintainer.py and session_report_downstream.py each
+import from here instead of each carrying their own copy.
+
+PATH RESOLUTION (ADR 0032): this module lives at
+plugins/dev-team/scripts/lib/, one level deeper than session_report.py
+itself, so its own __file__-relative path resolution needs one more
+`.parent` than the top-level CLI's did before the split.
+
+Stdlib only (Python 3.10+ floor, ADR 0031).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+_LIB_DIR = Path(__file__).resolve().parent
+if str(_LIB_DIR) not in sys.path:
+    sys.path.insert(0, str(_LIB_DIR))
+from session_log import classify, corrections, discovery, records, redact, signals
+
+_redact = redact.redact
+
+# --------------------------------------------------------------------------
+# Shared classification/signal vocabulary (session_log.classify/.signals,
+# issues #2042-#2044). Identical aliases in both predecessor scripts;
+# defined once here and used, unmodified, by both profiles' extract logic.
+# --------------------------------------------------------------------------
+_VERIFY_RE = classify.VERIFY_RE
+_CORRECTION_RE = classify.CORRECTION_RE
+_PERMISSION_RE = classify.PERMISSION_RE
+_OLDSTRING_RE = classify.OLDSTRING_RE
+_EDIT_TOOLS = signals.EDIT_TOOLS
+_GIT_GLOBAL_OPTS_WITH_ARG = classify.GIT_GLOBAL_OPTS_WITH_ARG
+_COMMIT_BYPASS_TOKENS = classify.COMMIT_BYPASS_TOKENS
+_statement_break_newlines = classify.statement_break_newlines
+_bash_segments = classify.bash_segments
+_is_git_commit_argv = classify.is_git_commit_argv
+_SAFE_NAME_RE = classify.SAFE_NAME_RE
+_UNSAFE_NAME = classify.UNSAFE_NAME
+_HARNESS_ATTRIBUTIONS = classify.HARNESS_ATTRIBUTIONS
+_MAIN_LABEL = "main"
+_UNATTRIBUTED_LABEL = "unattributed"
+_strip_ns = classify.strip_ns
+_text_of = classify.text_of
+
+_is_transcript_path = discovery.is_transcript_path
+_is_subagent_transcript = discovery.is_subagent_transcript
+# Two alias names for the same function (discovery.all_transcripts),
+# matching each predecessor's own naming so each profile's body is
+# otherwise unmodified.
+_all_transcripts_under = discovery.all_transcripts
+_all_transcripts = discovery.all_transcripts
+_sorted_paths = discovery.sorted_paths
+_relative_parts = discovery.relative_parts
+
+_track_tool_call = signals.track_tool_call
+_classify_tool_result = signals.classify_tool_result
+_track_edit = signals.track_edit
+_track_bash = signals.track_bash
+_new_thread = signals.new_thread
+_detect_correction_turn = signals.detect_correction_turn
+_new_agent_bucket = signals.new_agent_bucket
+_merge_agent_buckets = signals.merge_agent_buckets
+_finalize_agent_buckets = signals.finalize_agent_buckets
+_CONTEXT_TOKEN_FIELDS = signals.CONTEXT_TOKEN_FIELDS
+_AGENT_BUCKET_FIELDS = signals.AGENT_BUCKET_FIELDS
+_accumulate_skill_agent_signals = signals.accumulate_skill_agent_signals
+_slim = records.slim_by_name
+_iter_file_records = records.iter_file_records
+
+# --------------------------------------------------------------------------
+# Correction cause-data classification (session_log.corrections, #2013).
+# --------------------------------------------------------------------------
+_new_correction_context = corrections.new_context
+_observe_assistant_turn = corrections.observe_assistant_turn
+_classify_correction = corrections.classify_correction
+
+_VERSION_RE = re.compile(r"^[0-9A-Za-z._+-]{1,32}$")
+
+# --------------------------------------------------------------------------
+# Schema versioning. See session_report.py's module docstring: both profiles
+# bump to v3; the still-present predecessor scripts kept emitting v2
+# unchanged (both predecessors were later retired in #2048).
+# --------------------------------------------------------------------------
+_DIGEST_SCHEMA = "session-digest/v4"
+_SYNC_SCHEMA = "session-sync/v4"
+#: Sync-record schemas a reader accepts, oldest first. Exported so
+#: scripts/eval_rawlog.py (and any other reader) imports this constant
+#: instead of literal-matching a schema string — the ADR 0036 failure mode
+#: this guards against is a writer bumping the stamped schema while a
+#: reader still exact-matches the old one, which silently drops every
+#: record instead of erroring.
+SYNC_SCHEMAS = (
+    "session-sync/v1",
+    "session-sync/v2",
+    "session-sync/v3",
+    "session-sync/v4",
+)
+_DOWNSTREAM_SCHEMA = "downstream-session-report/v4"
+
+
+def _load_plugin_version(plugin_root: Path | None = None) -> str:
+    """Read `.claude-plugin/plugin.json`'s version. `plugin_root` defaults
+    to the dev-team plugin root (`plugins/dev-team/`) since both profiles
+    now resolve it the same way."""
+    if plugin_root is None:
+        plugin_root = Path(__file__).resolve().parent.parent.parent
+    manifest = Path(plugin_root) / ".claude-plugin" / "plugin.json"
+    try:
+        if manifest.stat().st_size > 64_000:
+            return "unknown"
+        data = json.loads(manifest.read_text(encoding="utf-8"))
+        version = data.get("version")
+        if isinstance(version, str) and _VERSION_RE.match(version):
+            return version
+    except (OSError, ValueError):
+        pass
+    return "unknown"
+
+
+def _new_correction_causes_state() -> dict:
+    """Fresh accumulator for correction cause-data (#2013): three label
+    Counters plus the ambiguous-shape tally, scoped to one `extract_*()`
+    call (across every transcript file it processes) -- NOT reset per
+    file, unlike `turn_context`/`active`/`thread` in each profile's loop."""
+    return {"by_what": Counter(), "by_component": Counter(), "by_shape": Counter(), "ambiguous": 0}
+
+
+def _record_correction_cause(state: dict, turn_context: dict, dispatch, text: str) -> None:
+    """Classify one detected correction turn and fold it into `state`."""
+    cause = _classify_correction(turn_context, dispatch, text)
+    state["by_what"][cause["what"]] += 1
+    state["by_component"][cause["component"]] += 1
+    state["by_shape"][cause["shape"]] += 1
+    if cause["confidence"] == "low":
+        state["ambiguous"] += 1
+
+
+def _finalize_correction_causes(state: dict, correction_turns: int) -> dict:
+    """The `accuracy.correction_causes` object: three sorted label
+    breakdowns plus the honest inference-share statistic (issue #2013
+    acceptance: "the inference share reported rather than hidden")."""
+    return {
+        "by_what": dict(sorted(state["by_what"].items())),
+        "by_component": dict(sorted(state["by_component"].items())),
+        "by_shape": dict(sorted(state["by_shape"].items())),
+        "ambiguous_share": round(state["ambiguous"] / correction_turns, 4)
+        if correction_turns
+        else 0.0,
+    }
+
+
+def _correction_rate_map(numerator: Counter, denominator: Counter) -> dict:
+    """Correction RATE per dispatch/invocation (issue #2013 acceptance:
+    "correction rate per dispatch is queryable by agent and by skill"),
+    mirroring `signals.finalize_agent_buckets`'s `context_per_dispatch`
+    pattern: a name with zero dispatches contributes no entry at all
+    (never a 0.0 that would misrank an uninvoked skill/agent as
+    "corrected 0% of the time")."""
+    return {
+        name: round(numerator.get(name, 0) / n, 4)
+        for name, n in sorted(denominator.items())
+        if n
+    }
+
+
+def _first_cwd(jsonl: Path) -> str | None:
+    try:
+        with jsonl.open(encoding="utf-8") as fh:
+            for _ in range(20):
+                line = fh.readline()
+                if not line:
+                    break
+                try:
+                    rec = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                rec_cwd = rec.get("cwd")
+                if isinstance(rec_cwd, str) and rec_cwd:
+                    return rec_cwd
+    except OSError:
+        pass
+    return None
+
+
+def resolve_session_plugin_version(session_id: str, boundary_events_path: Path) -> str:
+    """Per-session plugin_version resolver (#2018).
+
+    Correlates `session_id` against its own `boundary-events.jsonl` records
+    — a stream `hooks/lib/boundary_events.py::emit_boundary_event` stamps
+    LIVE, at hook-dispatch time, with the plugin version actually checked
+    out at that moment (`boundary_events._load_plugin_version`, ~line 79 of
+    that module). This is the one place a session's OWN version is
+    recoverable at all — transcripts themselves carry no version tag.
+
+    Choice — EARLIEST matching event by `ts` wins: a session's plugin
+    version is ordinarily constant for its whole duration (the checked-out
+    plugin doesn't change mid-session in the common case), so any matching
+    event would agree: earliest is picked because it is the version active
+    at the point the session first did dispatched work, the most
+    defensible single answer for "which release produced this session's
+    signals" when the session's activity happens to span more than one
+    event. Documented limitation, deliberately not solved here: a session
+    that runs `/dev-team:upgrade` (or otherwise changes the checked-out
+    plugin) mid-session will have GENUINELY DIFFERENT versions among its
+    own boundary-events, and this resolver reports only the first — there
+    is no single correct answer for a session that spans two releases, and
+    picking one is a documented trade-off, not an oversight.
+
+    Falls back to the explicit string `"unknown"` — NEVER a live
+    `plugin.json` read — when the file is missing, unreadable, has no
+    record for this `session_id`, or every matching record's own
+    `plugin_version` fails validation (`_VERSION_RE`, the same bound this
+    module applies to every other externally-supplied version string).
+
+    Shared by both profiles: the maintainer profile's `cmd_sync` uses it to
+    stamp per-session sync records with the version that actually produced
+    them (#2018); the downstream profile's own version-filter machinery
+    (`sessions_matching_plugin_version`/`_sessions_with_known_plugin_version`)
+    solves a related but different question and lives in
+    session_report_downstream.py."""
+    best_ts: str | None = None
+    best_version: str | None = None
+    for rec in _iter_file_records(boundary_events_path):
+        if str(rec.get("session_id") or "") != session_id:
+            continue
+        version = rec.get("plugin_version")
+        if not (isinstance(version, str) and _VERSION_RE.match(version)):
+            continue
+        ts = rec.get("ts")
+        if not isinstance(ts, str):
+            continue
+        if best_ts is None or ts < best_ts:
+            best_ts = ts
+            best_version = version
+    return best_version or "unknown"
+
+
+def load_registry(plugin_root: Path | None = None) -> dict:
+    """Enumerate shipped skills/agents so a digest/report can name
+    never-observed ones. `plugin_root` defaults the same way as
+    `_load_plugin_version` above."""
+    if plugin_root is None:
+        plugin_root = Path(__file__).resolve().parent.parent.parent
+    skills_dir = Path(plugin_root) / "skills"
+    agents_dir = Path(plugin_root) / "agents"
+    skills = sorted(p.name for p in skills_dir.iterdir()) if skills_dir.is_dir() else []
+    agents = (
+        sorted(p.stem for p in agents_dir.glob("*.md")) if agents_dir.is_dir() else []
+    )
+    return {"skills": skills, "agents": agents}

package/scripts/lib/settings_hook_guard.py

@@ -0,0 +1,157 @@
+"""Guard against machine-specific paths from graphify's Claude hook (#1367).
+
+`graphify install --project` (via its `claude install` step) appends
+PreToolUse hook entries to the target repo's shared, git-tracked
+`.claude/settings.json`, using the absolute path to the graphify binary on
+the installing machine (e.g. `/Users/alice/.local/bin/graphify`). If that
+file is committed, it bakes one developer's home-directory path into the
+repo and silently breaks for every other clone/teammate whose graphify
+binary lives somewhere else (different username, different install method —
+`uv tool`, `pipx`, `--user pip`, ... all resolve to different paths).
+
+This module scans `.claude/settings.json` for any `PreToolUse` hook entry
+that invokes graphify via an absolute filesystem path — instead of relying
+on PATH — and relocates it into `.claude/settings.local.json` (already
+gitignored, personal-machine scope). The scan is unconditional: it fixes a
+repo that already carries the pollution from a past install just as well as
+one guarding a fresh install, so `/project-init`/`/setup` can run it as a
+standing check rather than only wrapping a live installer call.
+
+No upstream flag exists to make `graphify` emit a bare, PATH-resolved
+command instead of an absolute path (checked against installed graphify
+0.9.8's `_resolve_graphify_exe`): it is a deliberate design choice, to keep
+the hook working in environments where the venv Scripts/ directory isn't on
+PATH (e.g. the VS Code Codex extension on Windows). Relocation, not an
+upstream flag, is therefore the fix.
+
+Stdlib-only, per ADR 0014/0015 (Python for cross-OS scripts).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import traceback
+from collections.abc import Callable
+from pathlib import Path
+
+_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]|^\\\\")
+_PRE_TOOL_USE_KEY = "PreToolUse"
+
+
+def _pre_tool_use(settings: dict) -> list[dict]:
+    return settings.get("hooks", {}).get(_PRE_TOOL_USE_KEY, [])
+
+
+def _load_json(path: Path) -> dict:
+    if not path.is_file():
+        return {}
+    text = path.read_text(encoding="utf-8").strip()
+    return json.loads(text) if text else {}
+
+
+def _dump_json(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def _command_is_absolute_graphify(command: str) -> bool:
+    token = command.strip().split(" ", 1)[0] if command.strip() else ""
+    if not token:
+        return False
+    is_absolute = token.startswith("/") or bool(_WINDOWS_ABSOLUTE.match(token))
+    if not is_absolute:
+        return False
+    basename = re.split(r"[\\/]", token)[-1]
+    if basename.lower().endswith(".exe"):
+        basename = basename[: -len(".exe")]
+    return basename.lower() == "graphify"
+
+
+def find_absolute_graphify_entries(settings: dict) -> list[dict]:
+    """Return `PreToolUse` entries whose command invokes graphify via an
+    absolute filesystem path rather than a bare, PATH-resolved `graphify`.
+    """
+    polluting = []
+    for entry in _pre_tool_use(settings):
+        commands = (h.get("command", "") for h in entry.get("hooks", []))
+        if any(_command_is_absolute_graphify(c) for c in commands):
+            polluting.append(entry)
+    return polluting
+
+
+def fix_settings(settings_path: Path, local_settings_path: Path) -> bool:
+    """Scan `settings_path` for absolute-path graphify hook entries and
+    relocate them into `local_settings_path`.
+
+    Returns True if anything was relocated, False if `settings_path` was
+    already clean (including when it doesn't exist).
+    """
+    settings = _load_json(settings_path)
+    polluting = find_absolute_graphify_entries(settings)
+    if not polluting:
+        return False
+
+    remaining = [e for e in _pre_tool_use(settings) if e not in polluting]
+    if remaining:
+        settings.setdefault("hooks", {})[_PRE_TOOL_USE_KEY] = remaining
+    else:
+        settings.get("hooks", {}).pop(_PRE_TOOL_USE_KEY, None)
+    _dump_json(settings_path, settings)
+
+    local_settings = _load_json(local_settings_path)
+    local_pre_tool_use = local_settings.setdefault("hooks", {}).setdefault(
+        _PRE_TOOL_USE_KEY, []
+    )
+    for entry in polluting:
+        if entry not in local_pre_tool_use:
+            local_pre_tool_use.append(entry)
+    _dump_json(local_settings_path, local_settings)
+
+    return True
+
+
+def run_install_with_guard(
+    settings_path: Path,
+    local_settings_path: Path,
+    installer: Callable[[], None],
+) -> bool:
+    """Run `installer()` against `settings_path`, then apply `fix_settings`.
+
+    Returns True if a machine-specific hook entry was detected and relocated
+    to `local_settings_path`, False if the installer's output was left as-is.
+
+    `fix_settings` runs even if `installer()` itself raises partway through a
+    write (e.g. it appends the polluting hook entry and then fails later in
+    the same run) — it happens in a `finally` block, and the original
+    exception still propagates once the relocation completes. No separate
+    `_read_or_none`-style helper is needed here: `fix_settings` already reads
+    `settings_path` through `_load_json`, which returns `{}` rather than
+    raising when the file doesn't exist, so it never assumes the file is
+    present.
+    """
+    was_relocated = False
+    installer_raised = False
+
+    try:
+        installer()
+    except BaseException:
+        # Bound to THIS call, not read from ambient `sys.exc_info()` inside
+        # `finally` below — see the identical pattern (and rationale) in
+        # claude_md_guard.py's run_install_with_guard.
+        installer_raised = True
+        raise
+    finally:
+        try:
+            was_relocated = fix_settings(settings_path, local_settings_path)
+        except Exception:
+            # A secondary failure while relocating must never displace
+            # installer()'s own in-flight exception — see the identical
+            # pattern (and rationale) in claude_md_guard.py's
+            # run_install_with_guard.
+            if installer_raised:
+                traceback.print_exc()
+            else:
+                raise
+
+    return was_relocated

package/scripts/lib/slug.py

@@ -0,0 +1,33 @@
+"""slug — shared repo-root slug derivation.
+
+Single source of truth for the kebab-safe, lowercase slug both
+`codebase_recon.py` (its `.claude/memory/recon-<slug>.json` artifact naming)
+and `orchestrator.py` (its `_recon_artifact_path` lookup of that same
+artifact) must agree on byte-for-byte — a drifted copy in either script
+would silently compute the wrong artifact path. Promoted from two
+hand-duplicated private copies (`codebase_recon.py::_derive_slug`,
+`orchestrator.py::_derive_recon_slug`) per issue #2068, following this
+directory's existing `from lib import ...` convention
+(`codebase_recon.py` already imports `deterministic_recon` this way).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+_NON_SLUG_CHARS = re.compile(r"[^a-z0-9._-]")
+_REPEATED_DASHES = re.compile(r"-{2,}")
+
+
+def derive_slug(root: Path) -> str:
+    """Derive a kebab-safe, lowercase slug from `root`'s resolved directory name.
+
+    Falls back to `"repo"` when the name has no slug-safe characters left
+    after stripping (e.g. an all-punctuation directory name).
+    """
+    name = root.resolve().name.lower()
+    name = _NON_SLUG_CHARS.sub("-", name)
+    name = _REPEATED_DASHES.sub("-", name)
+    name = name.strip("-")
+    return name or "repo"

package/scripts/lib/stub_extractors/__init__.py

@@ -0,0 +1,82 @@
+"""stub_extractors — per-language step-definition binding extractors for
+`gherkin_stub_merge.py` (issue #1421).
+
+A step-definition file is arbitrary general-purpose source code, unlike a
+`.feature` file's fixed, line-anchored Gherkin grammar — so "next marker line
+is the boundary" (the approach `gherkin_feature_merge.py` can safely use) is
+not safe here. Each binding's body must instead be bounded by an actual
+lexical scan of the source: brace/paren balance for JS/TS, Java, and C#
+(`jsts.py`/`java.py`/`csharp.py`), or — for Go, which registers a step via a
+single `sc.Step(pattern, funcName)` call referencing a separately-declared
+top-level function — a lookup of that referenced function's own
+brace-bounded body (`go.py`).
+
+The brace/paren balance count is **lexically aware**, not a naive character
+count: a `(`/`)`/`{`/`}` occurring inside a string/char literal or a
+line/block comment is skipped, not counted (`_common.py`), so a step body
+containing `const s = "}";` (or the C#/Go string-literal equivalents) can
+never be mistaken for a boundary. When a file's brackets genuinely don't
+balance, or a marker (a call, annotation, or attribute) has no attached,
+boundable body, `parse_existing_steps` returns a structural-error sentinel
+(`unbalanced-braces` / `dangling-annotation`) rather than guessing a span —
+refusing the write is the safety property this package exists to provide.
+
+Started as a single flat `_stub_extractors.py` module; split into this
+sub-package once the single-file draft passed the ~300-line size ceiling
+(plan-review structure finding) — one file per language plus one shared
+lexer module keeps each file's responsibility narrow, rather than
+reproducing `gherkin_feature_merge.py`'s five-plus-responsibility sprawl one
+import hop down.
+
+Reuses `lib._bdd_markers.MARKERS_BY_EXT` for the per-language pending-marker
+table (extension -> (language label, marker substrings)) rather than
+re-deriving it — a lib-level sibling import, not a reach into the parent
+`scripts/` entry-point layer (issue #1421 bug 5: this package used to import
+`gherkin_stub_gate`'s module-private `_MARKERS_BY_EXT` via a `sys.path`
+insertion reaching up to `scripts/`, the only module under `scripts/lib/` in
+the repo that imported from that parent layer).
+
+Stdlib-only. (ADR 0014/0015).
+"""
+
+from __future__ import annotations
+
+from _bdd_markers import MARKERS_BY_EXT as _PENDING_MARKERS_BY_EXT
+
+from . import csharp, go, java, jsts
+from ._common import (
+    ERROR_DANGLING_ANNOTATION,
+    ERROR_UNBALANCED_BRACES,
+    ParseResult,
+    StepBinding,
+)
+
+__all__ = [
+    "ERROR_DANGLING_ANNOTATION",
+    "ERROR_UNBALANCED_BRACES",
+    "ParseResult",
+    "StepBinding",
+    "parse_existing_steps",
+]
+
+_PARSERS_BY_LANGUAGE = {
+    "JS/TS": jsts.parse,
+    "Java": java.parse,
+    "C#": csharp.parse,
+    "Go": go.parse,
+}
+
+
+def parse_existing_steps(text: str, ext: str) -> ParseResult:
+    """Parse every step-definition binding in `text` for the language
+    implied by `ext` (matched case-insensitively, reusing
+    `lib._bdd_markers.MARKERS_BY_EXT`). Returns `ParseResult(bindings=[...],
+    error=None)` on success, or `ParseResult(bindings=None, error=<sentinel>)`
+    on the first structural problem encountered — never a partial, possibly-
+    wrong result."""
+    entry = _PENDING_MARKERS_BY_EXT.get(ext.lower())
+    if entry is None:
+        raise ValueError(f"unsupported step-definition extension: {ext!r}")
+    language, markers = entry
+    parser = _PARSERS_BY_LANGUAGE[language]
+    return parser(text, markers)