pi-ui-extend 1.0.40 → 1.0.42

package/skills/spec-wiki/evals/maintenance_regression.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""Deterministic regression checks for Spec Wiki maintenance primitives."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+CLI = Path(__file__).resolve().parents[1] / "scripts" / "spec_wiki.py"
+
+
+def run(cwd: Path, *args: str) -> str:
+    proc = subprocess.run(
+        ["python3", str(CLI), "--root", str(cwd), *args],
+        cwd=cwd,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )
+    if proc.returncode != 0:
+        raise AssertionError(f"command failed {args}:\nstdout={proc.stdout}\nstderr={proc.stderr}")
+    return proc.stdout
+
+
+def run_json(cwd: Path, *args: str) -> dict:
+    return json.loads(run(cwd, *args, "--json"))
+
+
+def git(cwd: Path, *args: str) -> None:
+    subprocess.run(["git", *args], cwd=cwd, check=True, stdout=subprocess.DEVNULL)
+
+
+def write(root: Path, rel: str, text: str) -> None:
+    path = root / rel
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def init_repo(root: Path) -> None:
+    git(root, "init", "-q")
+    git(root, "config", "user.email", "spec-wiki-eval@example.com")
+    git(root, "config", "user.name", "Spec Wiki Eval")
+
+
+def commit_all(root: Path, message: str = "baseline") -> None:
+    git(root, "add", ".")
+    git(root, "commit", "-qm", message)
+
+
+def test_relation_map_invalidates_verification(tmp: Path) -> None:
+    root = tmp / "relations"
+    root.mkdir()
+    init_repo(root)
+    write(root, "src/a.ts", "export const a = 1\n")
+    write(root, "src/b.ts", "export const b = 2\n")
+    write(
+        root,
+        "docs/contract.md",
+        """# Contract
+
+## Type
+As-is
+
+## Lifecycle
+Active
+
+## Behavior
+A is part of the current contract.
+
+## Related files
+- `src/a.ts`
+""",
+    )
+    commit_all(root)
+    run(
+        root,
+        "record",
+        "--path",
+        "docs/contract.md",
+        "--classification",
+        "spec",
+        "--type",
+        "as-is",
+        "--lifecycle",
+        "active",
+        "--confidence",
+        "high",
+        "--summary",
+        "Contract for A.",
+    )
+    run(root, "verify", "--path", "docs/contract.md")
+    audit = run_json(root, "audit")
+    assert audit["freshCount"] == 1
+
+    run(root, "relate", "--path", "docs/contract.md", "--add-code", "src/b.ts")
+    audit = run_json(root, "audit")
+    assert audit["freshCount"] == 0
+    status = next(item for item in audit["statuses"] if item["path"] == "docs/contract.md")
+    assert status["status"] == "inputs-changed"
+    assert "relations:changed" in status["reasons"]
+
+    run(root, "verify", "--path", "docs/contract.md")
+    assert run_json(root, "audit")["freshCount"] == 1
+    run(root, "relate", "--path", "docs/contract.md", "--remove-code", "src/b.ts")
+    status = next(item for item in run_json(root, "audit")["statuses"] if item["path"] == "docs/contract.md")
+    assert status["status"] == "inputs-changed"
+    assert status["reasons"] == ["relations:changed"]
+
+
+def test_impact_detects_coverage_gaps_and_moves(tmp: Path) -> None:
+    root = tmp / "impact"
+    root.mkdir()
+    init_repo(root)
+    write(root, "src/session.ts", "export function session() { return 'ok' }\n")
+    write(
+        root,
+        "docs/session-contract.md",
+        """# Session contract
+
+## Type
+As-is
+
+## Lifecycle
+Active
+
+## Behavior
+Session loading returns the current session.
+
+## Related files
+- `src/session.ts`
+""",
+    )
+    commit_all(root)
+    run(
+        root,
+        "record",
+        "--path",
+        "docs/session-contract.md",
+        "--classification",
+        "spec",
+        "--type",
+        "as-is",
+        "--lifecycle",
+        "active",
+        "--confidence",
+        "high",
+        "--summary",
+        "Session loading contract.",
+    )
+    run(root, "verify", "--path", "docs/session-contract.md")
+
+    git(root, "mv", "docs/session-contract.md", "docs/session-lifecycle.md")
+    with (root / "docs/session-lifecycle.md").open("a", encoding="utf-8") as handle:
+        handle.write("\nMoved without changing the behavioral contract.\n")
+    write(root, "src/refresh-worker.ts", "export const retryOnce = true\n")
+    write(root, "notes.txt", "deliberately low signal document\n")
+
+    impact = run_json(root, "impact")
+    assert ".spec-wiki/state.json" not in impact["changedPaths"]
+    assert any(
+        item.get("status") == "R"
+        and item.get("oldPath") == "docs/session-contract.md"
+        and item.get("path") == "docs/session-lifecycle.md"
+        for item in impact["changes"]
+    )
+    assert "src/refresh-worker.ts" in impact["uncoveredPaths"]
+    changed_docs = {item["path"]: item for item in impact["changedDocuments"]}
+    assert changed_docs["docs/session-lifecycle.md"]["requiresClassification"] is True
+    assert changed_docs["notes.txt"]["score"] == 0
+    assert "docs/session-contract.md" in impact["missingTrackedSpecs"]
+    assert impact["semanticSweepRequired"] is True
+
+
+def test_deep_audit_finds_low_signal_committed_docs(tmp: Path) -> None:
+    root = tmp / "deep"
+    root.mkdir()
+    init_repo(root)
+    write(
+        root,
+        "notes/capture.txt",
+        "Duplicate capture requests with the same key return the original result.\n",
+    )
+    write(root, "skills/example/evals/fixture.md", "# Contract\n\n## Behavior\nFixture only.\n")
+    commit_all(root)
+
+    normal = run_json(root, "discover", "--limit", "100")
+    assert "notes/capture.txt" not in {item["path"] for item in normal["candidates"]}
+    deep = run_json(root, "discover", "--all-unclassified", "--limit", "100")
+    paths = {item["path"] for item in deep["candidates"]}
+    assert "notes/capture.txt" in paths
+    assert "skills/example/evals/fixture.md" not in paths
+
+
+def test_relate_restores_source_explicit_relation(tmp: Path) -> None:
+    root = tmp / "supersession"
+    root.mkdir()
+    init_repo(root)
+    write(root, "docs/b.md", "# B\n")
+    write(
+        root,
+        "docs/a.md",
+        """# A contract
+
+## Type
+As-is
+
+## Lifecycle
+Active
+
+## Behavior
+A references B.
+
+[B](./b.md)
+""",
+    )
+    commit_all(root)
+    run(
+        root,
+        "record",
+        "--path",
+        "docs/a.md",
+        "--classification",
+        "spec",
+        "--type",
+        "as-is",
+        "--lifecycle",
+        "active",
+        "--confidence",
+        "high",
+        "--summary",
+        "A contract.",
+    )
+    run(root, "verify", "--path", "docs/a.md")
+    run(root, "relate", "--path", "docs/a.md", "--add-supersedes", "docs/b.md")
+    state = json.loads((root / ".spec-wiki/state.json").read_text(encoding="utf-8"))
+    relations = state["entries"]["docs/a.md"]["relations"]["specs"]
+    assert any(
+        item.get("path") == "docs/b.md"
+        and item.get("source") == "explicit"
+        and item.get("kind") == "related"
+        for item in relations
+    )
+    assert any(item.get("path") == "docs/b.md" and item.get("kind") == "supersedes" for item in relations)
+    run(root, "relate", "--path", "docs/a.md", "--remove-supersedes", "docs/b.md")
+    state = json.loads((root / ".spec-wiki/state.json").read_text(encoding="utf-8"))
+    relations = state["entries"]["docs/a.md"]["relations"]["specs"]
+    assert any(
+        item.get("path") == "docs/b.md"
+        and item.get("source") == "explicit"
+        and item.get("kind") == "related"
+        for item in relations
+    )
+    assert not any(item.get("kind") == "supersedes" for item in relations)
+
+
+def test_changed_non_primary_requires_reclassification(tmp: Path) -> None:
+    root = tmp / "reclassification"
+    root.mkdir()
+    init_repo(root)
+    write(root, "docs/info.md", "# Operator notes\n\nRun the service with --help.\n")
+    commit_all(root)
+    run(
+        root,
+        "record",
+        "--path",
+        "docs/info.md",
+        "--classification",
+        "guide",
+        "--summary",
+        "Operator notes.",
+    )
+    write(
+        root,
+        "docs/info.md",
+        """# Request contract
+
+## Type
+As-is
+
+## Lifecycle
+Active
+
+## Behavior
+Duplicate requests return the original result.
+""",
+    )
+    impact = run_json(root, "impact", "docs/info.md")
+    changed = next(item for item in impact["changedDocuments"] if item["path"] == "docs/info.md")
+    assert changed["knownClassification"] == "guide"
+    assert changed["requiresClassification"] is False
+    assert changed["requiresReclassification"] is True
+    assert impact["semanticSweepRequired"] is True
+
+
+def main() -> None:
+    with tempfile.TemporaryDirectory(prefix="spec-wiki-maint-") as directory:
+        tmp = Path(directory)
+        test_relation_map_invalidates_verification(tmp)
+        test_impact_detects_coverage_gaps_and_moves(tmp)
+        test_deep_audit_finds_low_signal_committed_docs(tmp)
+        test_relate_restores_source_explicit_relation(tmp)
+        test_changed_non_primary_requires_reclassification(tmp)
+    print("maintenance regression: 5/5 passed")
+
+
+if __name__ == "__main__":
+    main()

package/skills/spec-wiki/evals/retrieval-eval.md

@@ -0,0 +1,37 @@
+# Retrieval regression protocol
+
+Use `retrieval-evals.json` to check the query router after search/index changes.
+The cases are intentionally mixed: direct keywords, natural paraphrases,
+Russian requests, synonyms, exact code paths, and symbols.
+
+## Guard against answer leakage
+
+The retrieval agent must receive only each case `id` and `query`. Do not expose
+`expectedAny` until after it returns candidate paths.
+
+## Stage 1: deterministic candidate generation
+
+Run `spec_wiki.py search` against the current `.spec-wiki/state.json` and record
+top-1/top-3/top-5 plus `confidence` / `catalogFallbackRecommended`. This stage is
+expected to be imperfect for semantic paraphrases; its job is cheap candidate
+generation and reliable low-confidence detection.
+
+## Stage 2: full skill workflow
+
+Have a fresh agent read only `SKILL.md` and `references/query.md`, then for each
+query:
+
+1. create the in-model retrieval bundle;
+2. call relation-aware `search` with `--also` aliases;
+3. semantically rerank compact result metadata;
+4. when fallback is recommended or intent coverage is visibly incomplete, use
+   only `.spec-wiki/index.md` as the semantic catalog fallback;
+5. return up to three primary spec paths without opening the primary specs.
+
+Compare those paths with `expectedAny` only after the run. A case passes top-k
+when any expected path appears in the first k selections.
+
+Current target for this repository: at least 95% top-1 and 100% top-3/recall.
+Do not tune expected paths after seeing a run merely to improve the score; change
+an expectation only when independent review shows that more than one primary
+spec is genuinely defensible.

package/skills/spec-wiki/evals/retrieval-evals.json

@@ -0,0 +1,37 @@
+{
+  "version": 1,
+  "description": "Live retrieval regression cases for the host repository. Expected paths are chosen before running the search; use expectedAny when more than one primary spec is defensible.",
+  "cases": [
+    {"id":"kw-dcp","category":"keyword","query":"dcp persistence recovery","expectedAny":["specs/dcp.md"]},
+    {"id":"kw-desktop-markdown","category":"keyword","query":"desktop markdown mermaid file preview","expectedAny":["specs/desktop-markdown-rendering.md"]},
+    {"id":"kw-project-agents","category":"keyword","query":"project local agents .pi/agents","expectedAny":["specs/project-agents-dir.md"]},
+    {"id":"kw-question-tab","category":"keyword","query":"question inactive tab","expectedAny":["specs/question-inactive-tab.md"]},
+    {"id":"kw-retry-toast","category":"keyword","query":"retry toast action","expectedAny":["specs/retry-toast-action.md"]},
+    {"id":"kw-openai-usage","category":"keyword","query":"openai codex token refresh usage","expectedAny":["specs/openai-codex-usage-refresh.md"]},
+
+    {"id":"para-recovery","category":"paraphrase","query":"recover old conversation after compaction","expectedAny":["external/pi-tools-suite/docs/session-recovery.md"]},
+    {"id":"para-browser-qa","category":"paraphrase","query":"browser qa credentials origin isolation redaction","expectedAny":["external/pi-tools-suite/docs/browser-qa-subagent.md"]},
+    {"id":"para-concurrency","category":"paraphrase","query":"async callback updates the wrong session tab","expectedAny":["docs/concurrency.md"]},
+    {"id":"para-cache","category":"paraphrase","query":"keep tool output stable for prompt caching","expectedAny":["specs/dcp-provider-cache-stability.md"]},
+    {"id":"para-window","category":"paraphrase","query":"window opens offscreen after monitor change","expectedAny":["specs/desktop-window-state.md"]},
+    {"id":"para-copy","category":"paraphrase","query":"copy last assistant response slash command","expectedAny":["specs/desktop-slash-commands.md"]},
+    {"id":"para-task-jsonc","category":"paraphrase","query":"task storage comments jsonc","expectedAny":["docs/desktop-task-manager.md"]},
+    {"id":"para-wrapped-link","category":"paraphrase","query":"click file uri in wrapped markdown","expectedAny":["specs/clickable-markdown-links.md"]},
+    {"id":"para-zed","category":"paraphrase","query":"open source file in zed at line","expectedAny":["specs/file-link-opening.md"]},
+
+    {"id":"ru-recovery","category":"russian","query":"как восстанавливается история после сжатия контекста","expectedAny":["external/pi-tools-suite/docs/session-recovery.md"]},
+    {"id":"ru-zed","category":"russian","query":"как открыть ссылку на файл в zed","expectedAny":["specs/file-link-opening.md"]},
+    {"id":"ru-question-tab","category":"russian","query":"что происходит с вопросом в неактивной вкладке","expectedAny":["specs/question-inactive-tab.md"]},
+    {"id":"ru-tasks","category":"russian","query":"где хранятся задачи проекта и поддерживаются ли комментарии","expectedAny":["docs/desktop-task-manager.md"]},
+    {"id":"ru-provider-seen","category":"russian","query":"почему старый tool result нельзя удалять до отправки провайдеру","expectedAny":["specs/dcp-provider-cache-stability.md"]},
+
+    {"id":"syn-auth","category":"synonym","query":"oauth account rotation when quota exhausted","expectedAny":["specs/antigravity-auth.md"]},
+    {"id":"syn-background-agent","category":"synonym","query":"cancel a running background agent","expectedAny":["specs/async-subagents.md"]},
+
+    {"id":"path-openai","category":"path","query":"src/app/model/model-usage-status.ts","expectedAny":["specs/openai-codex-usage-refresh.md"]},
+    {"id":"path-dcp-journal","category":"path","query":"external/pi-tools-suite/src/dcp/journal.ts","expectedAny":["specs/dcp.md","specs/dcp-reliability-mechanisms.md","specs/dcp-provider-cache-stability.md"]},
+
+    {"id":"symbol-sidebar","category":"symbol","query":"WorkspaceSidebar.svelte tasks","expectedAny":["docs/desktop-task-manager.md"]},
+    {"id":"symbol-question-bridge","category":"symbol","query":"ui-request-bridge question","expectedAny":["specs/desktop-question-tool.md"]}
+  ]
+}

package/skills/spec-wiki/evals/trigger-evals.json

@@ -0,0 +1,94 @@
+[
+  {
+    "query": "Before we touch src/auth/session.ts, write a compact spec for the new refresh-token rotation behavior. It changes persistence and the public login flow, and I want the expected behavior and verification path clear before implementation.",
+    "should_trigger": true
+  },
+  {
+    "query": "We have a pile of design docs under docs/, architecture/, and feature folders with no central specs directory. Find which ones are actual behavioral specs and build a small LLM-friendly project spec index without treating existing overview pages as source-of-truth.",
+    "should_trigger": true
+  },
+  {
+    "query": "src/jobs/runner.ts changed yesterday. Can you tell me which project specs may now be stale and check whether the documented cancellation/retry behavior still matches code and tests?",
+    "should_trigger": true
+  },
+  {
+    "query": "Which of our specs defines desktop session restore behavior? Please answer from the primary spec rather than scanning every markdown file, and warn me if the relevant spec looks stale.",
+    "should_trigger": true
+  },
+  {
+    "query": "The bug fix changes an externally visible CLI exit-code contract. Update the existing spec to describe the intended behavior, edge cases and compatibility, but don't implement the code yet.",
+    "should_trigger": true
+  },
+  {
+    "query": "Document the current as-is behavior of our webhook retry loop from code and tests. I'm especially worried the README is stale, so separate confirmed behavior from inference and unknowns.",
+    "should_trigger": true
+  },
+  {
+    "query": "We renamed several contract docs and moved implementation files. Refresh the project's spec knowledge base, repair missing paths, and show me which specs still need semantic review.",
+    "should_trigger": true
+  },
+  {
+    "query": "What do the project requirements say about idempotency for payment capture? There are multiple RFCs and an overview; find the primary spec and answer with its source path.",
+    "should_trigger": true
+  },
+  {
+    "query": "I'm about to change the on-disk cache schema. Check if we already have a spec for that behavior; if not, create a lightweight change spec before touching implementation.",
+    "should_trigger": true
+  },
+  {
+    "query": "Inventory the behavioral contracts in this unfamiliar repo and tell me which ones are affected by the changes in src/api and src/storage. Specs may be scattered and inconsistently named.",
+    "should_trigger": true
+  },
+  {
+    "query": "Implement background session refresh so a transient token-load failure is retried exactly once and a second failure still propagates. Add the code and tests and verify the change is complete.",
+    "should_trigger": true
+  },
+  {
+    "query": "I added a new src/auth/refresh-policy.ts implementation surface for an existing login behavior. Finish the change and make sure any project contracts or knowledge that should track the new file stay current.",
+    "should_trigger": true
+  },
+  {
+    "query": "Refactor src/utils/string.ts to use a helper function. This is a mechanical cleanup with no externally visible behavior change; just update the code and tests.",
+    "should_trigger": false
+  },
+  {
+    "query": "Write an ADR comparing Postgres and SQLite for a future analytics service. I want design tradeoffs and a decision record, not a behavior contract or spec audit.",
+    "should_trigger": false
+  },
+  {
+    "query": "Rewrite docs/getting-started.md so new users understand how to log in and run the CLI. Keep it friendly and add examples.",
+    "should_trigger": false
+  },
+  {
+    "query": "Search all markdown files for the exact string `refresh_token` and show me the matching lines. Don't interpret or update any documentation.",
+    "should_trigger": false
+  },
+  {
+    "query": "Trace callers of createSession and tell me what code modules would be affected by changing its return type. I only need code impact analysis, not documentation work.",
+    "should_trigger": false
+  },
+  {
+    "query": "Generate API reference pages from our OpenAPI JSON and publish them under docs/generated/. The schema is already authoritative; don't create or review project specs.",
+    "should_trigger": false
+  },
+  {
+    "query": "Summarize architecture/rfc-019.md for me in five bullets. I only want a summary of this one file, not a spec validation or project knowledge index.",
+    "should_trigger": false
+  },
+  {
+    "query": "Fix the typo in the error message in src/ui/toast.ts and adjust the snapshot. No contract or compatibility behavior changes.",
+    "should_trigger": false
+  },
+  {
+    "query": "Create a migration runbook for operators moving production from Redis 6 to Redis 7, including rollback commands and monitoring checks.",
+    "should_trigger": false
+  },
+  {
+    "query": "Compare these two external W3C specifications and explain the differences in browser behavior. This isn't about specs stored in our project.",
+    "should_trigger": false
+  },
+  {
+    "query": "Add a debug-only telemetry label to an internal helper and update its unit test. It must not change any user-visible behavior, persistence, API, retry, or compatibility contract.",
+    "should_trigger": false
+  }
+]

package/skills/spec-wiki/references/bootstrap.md

@@ -0,0 +1,100 @@
+# Bootstrap the Spec Wiki
+
+Use for repository-wide discovery when no useful `.spec-wiki/state.json` exists,
+or when the user explicitly asks to inventory scattered specifications.
+
+The repository may have no `specs/` directory, no frontmatter, inconsistent
+formats, and multiple documentation roots.
+
+## Workflow
+
+1. Run `discover --limit 40`.
+2. Read only the high-signal shortlist.
+3. Semantically classify each candidate:
+   - `spec` — primary normative/descriptive specification;
+   - `spec-like` — useful spec knowledge with mixed/ambiguous purpose;
+   - `meta-index` — overview, inventory, catalog, TOC, generated index;
+   - `design-only` — design/decision material without enough behavioral contract;
+   - `guide` — tutorial/runbook/usage documentation;
+   - `other` — not useful spec knowledge.
+   Treat files under eval/fixture/example/testdata trees, skill resources, and
+   agent-role definition directories as examples/runtime configuration by
+   default, not project primary specs, unless repository evidence explicitly
+   says that document itself is normative project behavior.
+
+   Location is not a reason to downgrade a document. A document outside a
+   `specs/` directory that explicitly declares its behavioral Type/Lifecycle
+   and defines testable behavior/contracts/invariants should normally be
+   `spec`. Reserve `spec-like` for genuinely mixed-purpose or ambiguous
+   documents, not for clear subsystem contracts living under `docs/`.
+4. `record` every reviewed classification. For `spec`/`spec-like`, provide a
+   concise summary, type, lifecycle, confidence, and a few retrieval topics.
+   Classification/indexing is **not** semantic verification: bootstrap entries
+   normally remain `unverified` until a focused code/test review occurs.
+5. After recording a batch, rerun `discover --limit 40` from offset 0; unchanged
+   reviewed documents drop from the default shortlist.
+6. Continue until useful candidates are exhausted or coverage becomes too broad.
+   Report incomplete coverage rather than reading hundreds of weak candidates.
+7. Run `render` and `validate`.
+
+Do not call `verify` merely because a document was read during discovery. Call
+it only after checking the document's behavioral claims against the relevant
+implementation/tests (or other authoritative evidence for a proposed contract).
+
+## What counts as a primary spec
+
+A primary spec describes current or intended behavior, requirements, contracts,
+invariants, externally observable semantics, acceptance criteria, compatibility
+rules, or similarly testable expectations.
+
+Classification is semantic. Filename, folder, headings, and links are signals,
+not requirements.
+
+## Lifecycle
+
+Classify primary documents separately from their as-is/change type:
+
+- `active` — current authoritative contract;
+- `proposed` — intended contract not yet established as current behavior;
+- `historical` — useful milestone/history, not current precedence;
+- `superseded` — replaced by a newer primary spec; add `--superseded-by`;
+- `unknown` — lifecycle cannot be established safely.
+
+Useful default reasoning:
+
+- an as-is/current-behavior contract is normally `active` unless clearly
+  historical;
+- a change document whose behavior is already implemented and still governs the
+  system is normally `active`;
+- an intended/not-yet-implemented change is `proposed`;
+- milestone documents whose former non-goals/features were later superseded are
+  `historical` unless a newer document explicitly replaces their whole contract;
+- use `superseded` only when a specific replacement exists and record that edge.
+
+The CLI requires explicit `--type` and `--lifecycle` on the first primary-spec
+record so omission cannot silently become precedence metadata.
+
+Use `--supersedes` / `--superseded-by` for known precedence instead of leaving
+overlapping documents as coequal active specs.
+
+## Meta/index exclusion
+
+Do not index derived inventories as primary specs. They are useful only to find
+primary documents missed by heuristics. This avoids recursive knowledge such as:
+
+`spec -> old index summary -> new wiki summary`.
+
+## Relationships
+
+The CLI extracts resolvable source links/paths as `explicit` relationships. Add
+`--code` or `--related-spec` only for relationships actually established from
+repository evidence; these are recorded as `inferred`.
+
+The resolver tries the source directory, nearest package/project roots, and the
+repository root. Review `unresolvedReferences` after bootstrap: they are path-like
+hints that could not be resolved deterministically and may need an inferred
+relation or a source-doc fix.
+
+Use project-relative path identity, never numeric prefixes as unique IDs.
+
+For exact command/config/state details, read `index-schema.md`.