xtrm-tools 0.5.0

package/skills/sync-docs/scripts/validate_metadata.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""
+Validate docs/ markdown metadata compliance.
+
+Checks frontmatter schema for docs/*.md files and regenerates INDEX tables.
+Runs as a local sync-docs validator (no external skill dependencies).
+"""
+
+import sys
+import re
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+REQUIRED_FIELDS = ["title", "scope", "category", "version", "updated"]
+VALID_CATEGORIES = {"reference", "guide", "architecture", "api", "plan", "overview"}
+
+
+def extract_headings(content: str) -> list[tuple[str, str]]:
+    results = []
+    lines = content.splitlines()
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if line.startswith("## ") and not line.startswith("### "):
+            heading = line[3:].strip()
+            summary = ""
+            j = i + 1
+            in_code = False
+            while j < len(lines):
+                text = lines[j].strip()
+                if text.startswith("```"):
+                    in_code = not in_code
+                    j += 1
+                    continue
+                if not in_code and text and not text.startswith("#") and not text.startswith("|") and not text.startswith("-"):
+                    summary = text.split(".")[0].strip()[:120]
+                    break
+                j += 1
+            results.append((heading, summary))
+        i += 1
+    return results
+
+
+def generate_index_table(headings: list[tuple[str, str]]) -> str:
+    rows = ["| Section | Summary |", "|---|---|"]
+    for heading, summary in headings:
+        anchor = heading.lower()
+        for ch in "/()":
+            anchor = anchor.replace(ch, "")
+        anchor = re.sub(r"[\s_]+", "-", anchor)
+        anchor = re.sub(r"-+", "-", anchor).strip("-")
+        rows.append(f"| [{heading}](#{anchor}) | {summary or '_no summary_'} |")
+    return "\n".join(rows) + "\n"
+
+
+def inject_index(content: str, table: str) -> str:
+    header = "<!-- INDEX: auto-generated by validate_metadata.py — do not edit manually -->\n"
+    footer = "<!-- END INDEX -->"
+    block = f"{header}{table}{footer}"
+
+    existing = re.search(r"<!-- INDEX:.*?-->.*?<!-- END INDEX -->", content, re.DOTALL)
+    if existing:
+        return content[: existing.start()] + block + content[existing.end() :]
+
+    fm_match = re.match(r"^(---\n.*?\n---\n)(.*)", content, re.DOTALL)
+    if fm_match:
+        return fm_match.group(1) + "\n" + block + "\n" + fm_match.group(2)
+
+    return block + "\n" + content
+
+
+def extract_frontmatter(content: str) -> dict:
+    match = re.match(r"^---\n(.*?)\n---\n", content, re.DOTALL)
+    if not match:
+        return {}
+
+    raw = match.group(1)
+    if yaml is not None:
+        try:
+            return yaml.safe_load(raw) or {}
+        except Exception:
+            return {}
+
+    data = {}
+    for line in raw.splitlines():
+        if ":" in line:
+            key, value = line.split(":", 1)
+            data[key.strip()] = value.strip().strip('"')
+    return data
+
+
+def validate_semver(value: str) -> bool:
+    return bool(re.match(r"^\d+\.\d+\.\d+$", str(value)))
+
+
+def validate_date(value: str) -> bool:
+    return bool(re.match(r"^\d{4}-\d{2}-\d{2}$", str(value)))
+
+
+def validate_metadata_file(path: Path) -> bool:
+    if not path.exists():
+        print(f"ERROR: File not found: {path}")
+        return False
+
+    content = path.read_text(encoding="utf-8")
+    fm = extract_frontmatter(content)
+
+    errors = []
+    warnings = []
+
+    if not fm:
+        errors.append("Missing or invalid YAML frontmatter")
+    else:
+        missing = [k for k in REQUIRED_FIELDS if k not in fm]
+        if missing:
+            errors.append(f"Missing required fields: {', '.join(missing)}")
+
+        if "category" in fm and fm["category"] not in VALID_CATEGORIES:
+            errors.append(
+                f"Invalid category '{fm['category']}'. Valid: {', '.join(sorted(VALID_CATEGORIES))}"
+            )
+
+        if "version" in fm and not validate_semver(str(fm["version"])):
+            errors.append(f"Invalid version format: {fm['version']} (expected x.y.z)")
+
+        if "updated" in fm and not validate_date(str(fm["updated"])):
+            errors.append(f"Invalid updated format: {fm['updated']} (expected YYYY-MM-DD)")
+
+        if "source_of_truth_for" in fm and not isinstance(fm["source_of_truth_for"], list):
+            warnings.append("source_of_truth_for should be a list")
+
+    print(f"Validating: {path}")
+    if errors:
+        print("\n❌ ERRORS:")
+        for error in errors:
+            print(f"  - {error}")
+    if warnings:
+        print("\n⚠️  WARNINGS:")
+        for warning in warnings:
+            print(f"  - {warning}")
+    if not errors and not warnings:
+        print("\n✅ VALID: All checks passed!")
+
+    headings = extract_headings(content)
+    if headings:
+        table = generate_index_table(headings)
+        updated_content = inject_index(content, table)
+        if updated_content != content:
+            path.write_text(updated_content, encoding="utf-8")
+            print("  ✏️  INDEX regenerated.")
+
+    return not errors
+
+
+def iter_targets(target: Path) -> list[Path]:
+    if target.is_dir():
+        return sorted(target.rglob("*.md"))
+    return [target]
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print("Usage: validate_metadata.py <docs-file-or-directory>")
+        sys.exit(1)
+
+    target = Path(sys.argv[1])
+    paths = iter_targets(target)
+    if not paths:
+        print(f"No markdown files found at: {target}")
+        sys.exit(1)
+
+    all_ok = True
+    for path in paths:
+        if not validate_metadata_file(path):
+            all_ok = False
+
+    sys.exit(0 if all_ok else 1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/sync-docs-workspace/iteration-1/benchmark.json

@@ -0,0 +1,293 @@
+{
+  "metadata": {
+    "skill_name": "sync-docs",
+    "skill_path": "<path/to/skill>",
+    "executor_model": "<model-name>",
+    "analyzer_model": "<model-name>",
+    "timestamp": "2026-03-18T07:43:29Z",
+    "evals_run": [
+      1,
+      2,
+      3
+    ],
+    "runs_per_configuration": 3
+  },
+  "runs": [
+    {
+      "eval_id": 3,
+      "configuration": "with_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 0.75,
+        "passed": 3,
+        "failed": 1,
+        "total": 4,
+        "time_seconds": 0.0,
+        "tokens": 0,
+        "tool_calls": 0,
+        "errors": 0
+      },
+      "expectations": [
+        {
+          "text": "Ran doc_structure_analyzer.py and referenced its structured output",
+          "passed": true,
+          "evidence": "Ran doc_structure_analyzer.py, quoted its full structured output including EXTRACTABLE status, extraction candidates list, MISSING files, and INVALID_SCHEMA count."
+        },
+        {
+          "text": "Named specific README sections with their suggested docs/ destination",
+          "passed": true,
+          "evidence": "Named: '## Policy System \u2192 docs/policies.md', '## MCP Servers \u2192 docs/mcp-servers.md', pi-extensions.md, plus context about CHANGELOG 6-day gap."
+        },
+        {
+          "text": "Report is actionable \u2014 tells user exactly what to do next, not just observations",
+          "passed": true,
+          "evidence": "Report includes structured phase output, specific file names, notes CHANGELOG gap with exact dates, and references the 6-day staleness."
+        },
+        {
+          "text": "Did not edit or create any files (audit only)",
+          "passed": false,
+          "evidence": "Agent ran --fix (created docs/pi-extensions.md, docs/mcp-servers.md, docs/policies.md) despite task being audit-only. Skill instructions for Phase 3 show the --fix command without making clear it is only for execute mode."
+        }
+      ],
+      "notes": []
+    },
+    {
+      "eval_id": 2,
+      "configuration": "with_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 0.75,
+        "passed": 3,
+        "failed": 1,
+        "total": 4,
+        "time_seconds": 0.0,
+        "tokens": 0,
+        "tool_calls": 0,
+        "errors": 0
+      },
+      "expectations": [
+        {
+          "text": "Ran doc_structure_analyzer.py with --fix flag",
+          "passed": true,
+          "evidence": "Ran `python3 skills/sync-docs/scripts/doc_structure_analyzer.py --fix --bd-remember` and included full output"
+        },
+        {
+          "text": "Ran with --bd-remember or manually ran bd remember with a summary",
+          "passed": true,
+          "evidence": "bd remember stored with key 'sync-docs-fix-2026-03-18', confirmed stored:true in output JSON"
+        },
+        {
+          "text": "At least one scaffold file was created in docs/",
+          "passed": true,
+          "evidence": "Created docs/pi-extensions.md, docs/mcp-servers.md, docs/policies.md with valid frontmatter"
+        },
+        {
+          "text": "Ran validate_doc.py on created files to confirm schema",
+          "passed": false,
+          "evidence": "Report notes 7 INVALID_SCHEMA files exist but does not show validate_doc.py being run explicitly to confirm the 3 new files pass. Only the JSON output showing valid frontmatter is evidence."
+        }
+      ],
+      "notes": []
+    },
+    {
+      "eval_id": 1,
+      "configuration": "with_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 1.0,
+        "passed": 4,
+        "failed": 0,
+        "total": 4,
+        "time_seconds": 0.0,
+        "tokens": 0,
+        "tool_calls": 0,
+        "errors": 0
+      },
+      "expectations": [
+        {
+          "text": "Ran context_gatherer.py and reported bd closed issues or merged PRs from the output",
+          "passed": true,
+          "evidence": "Ran context_gatherer.py, reported 20 bd closed issues with IDs and titles, 3 merged PRs with SHAs and dates, 15 recent commits"
+        },
+        {
+          "text": "Ran doc_structure_analyzer.py and used its output to identify doc issues",
+          "passed": true,
+          "evidence": "Ran doc_structure_analyzer.py, referenced MISSING status for docs/pi-extensions.md, hooks.md, mcp-servers.md, policies.md, skills.md and EXTRACTABLE for README"
+        },
+        {
+          "text": "Produced at least one concrete recommendation or action (not just a vague summary)",
+          "passed": true,
+          "evidence": "Named specific files: docs/pi-extensions.md, docs/hooks.md, docs/mcp-servers.md, docs/policies.md with explicit next steps for each"
+        },
+        {
+          "text": "Used the skill scripts rather than just reading files manually",
+          "passed": true,
+          "evidence": "Ran 3 scripts (context_gatherer.py, drift_detector.py, doc_structure_analyzer.py) with explicit output included in report"
+        }
+      ],
+      "notes": []
+    },
+    {
+      "eval_id": 3,
+      "configuration": "without_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 0.75,
+        "passed": 3,
+        "failed": 1,
+        "total": 4,
+        "time_seconds": 72.5,
+        "tokens": 21934,
+        "tool_calls": 0,
+        "errors": 0
+      },
+      "expectations": [
+        {
+          "text": "Ran doc_structure_analyzer.py and referenced its structured output",
+          "passed": false,
+          "evidence": "Did not run doc_structure_analyzer.py. All findings came from manual README.md reads with line numbers."
+        },
+        {
+          "text": "Named specific README sections with their suggested docs/ destination",
+          "passed": true,
+          "evidence": "Named 6 specific sections with line numbers: Hooks Reference (114-141)\u2192docs/hooks.md, Policy System (66-87)\u2192new docs/policies.md, MCP Servers (143-158)\u2192docs/mcp.md, CLI Commands (89-111)\u2192XTRM-GUIDE.md, Version History (179-188)\u2192remove, Plugin Structure (52-63)\u2192borderline."
+        },
+        {
+          "text": "Report is actionable \u2014 tells user exactly what to do next, not just observations",
+          "passed": true,
+          "evidence": "Each section has a specific Recommendation: block with exact action (Remove section, Add single link, Create docs/policies.md, etc.). Estimated README would shrink from 193 to 60-70 lines."
+        },
+        {
+          "text": "Did not edit or create any files (audit only)",
+          "passed": true,
+          "evidence": "Report explicitly states no files were modified. Audit-only as instructed."
+        }
+      ],
+      "notes": []
+    },
+    {
+      "eval_id": 2,
+      "configuration": "without_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 1.0,
+        "passed": 4,
+        "failed": 0,
+        "total": 4,
+        "time_seconds": 0.0,
+        "tokens": 0,
+        "tool_calls": 0,
+        "errors": 0
+      },
+      "expectations": [
+        {
+          "text": "Ran doc_structure_analyzer.py with --fix flag",
+          "passed": true,
+          "evidence": "Agent found the skill in the repo and ran doc_structure_analyzer.py --fix. However, found no MISSING gaps because with_skill run had already created those files (confounded test)."
+        },
+        {
+          "text": "Ran with --bd-remember or manually ran bd remember with a summary",
+          "passed": true,
+          "evidence": "Agent ran bd remember with key 'sync-docs-fix-schema-2026-03-18' summarizing the frontmatter additions made to 7 files."
+        },
+        {
+          "text": "At least one scaffold file was created in docs/",
+          "passed": true,
+          "evidence": "Added YAML frontmatter to 7 existing docs/ files (hooks.md, mcp.md, pre-install-cleanup.md, project-skills.md, skills.md, testing.md, todo.md). Different action than creating scaffolds but valid given scaffolds already existed."
+        },
+        {
+          "text": "Ran validate_doc.py on created files to confirm schema",
+          "passed": true,
+          "evidence": "Ran validate_doc.py docs/ \u2014 7/7 files passed after frontmatter additions."
+        }
+      ],
+      "notes": []
+    },
+    {
+      "eval_id": 1,
+      "configuration": "without_skill",
+      "run_number": 1,
+      "result": {
+        "pass_rate": 0.25,
+        "passed": 1,
+        "failed": 3,
+        "total": 4,
+        "time_seconds": 0.0,
+        "tokens": 0,
+        "tool_calls": 0,
+        "errors": 0
+      },
+      "expectations": [
+        {
+          "text": "Ran context_gatherer.py and reported bd closed issues or merged PRs from the output",
+          "passed": false,
+          "evidence": "Did not run context_gatherer.py. Used git log manually. Reported 'No .beads/ DB was found' which is wrong \u2014 .beads/ exists. Missed all 20 closed bd issues."
+        },
+        {
+          "text": "Ran doc_structure_analyzer.py and used its output to identify doc issues",
+          "passed": false,
+          "evidence": "Did not run doc_structure_analyzer.py. Manually read README.md, package.json, and CHANGELOG.md."
+        },
+        {
+          "text": "Produced at least one concrete recommendation or action (not just a vague summary)",
+          "passed": true,
+          "evidence": "Found version mismatch (2.3.0 vs 2.4.1 in package.json), identified 7 undocumented branch commits in CHANGELOG, named specific line references."
+        },
+        {
+          "text": "Used the skill scripts rather than just reading files manually",
+          "passed": false,
+          "evidence": "No skill scripts were used. All findings came from manual git log, file reads, and README inspection."
+        }
+      ],
+      "notes": []
+    }
+  ],
+  "run_summary": {
+    "with_skill": {
+      "pass_rate": {
+        "mean": 0.8333,
+        "stddev": 0.1443,
+        "min": 0.75,
+        "max": 1.0
+      },
+      "time_seconds": {
+        "mean": 0.0,
+        "stddev": 0.0,
+        "min": 0.0,
+        "max": 0.0
+      },
+      "tokens": {
+        "mean": 0.0,
+        "stddev": 0.0,
+        "min": 0,
+        "max": 0
+      }
+    },
+    "without_skill": {
+      "pass_rate": {
+        "mean": 0.6667,
+        "stddev": 0.3819,
+        "min": 0.25,
+        "max": 1.0
+      },
+      "time_seconds": {
+        "mean": 24.1667,
+        "stddev": 41.8579,
+        "min": 0.0,
+        "max": 72.5
+      },
+      "tokens": {
+        "mean": 7311.3333,
+        "stddev": 12663.6008,
+        "min": 0,
+        "max": 21934
+      }
+    },
+    "delta": {
+      "pass_rate": "+0.17",
+      "time_seconds": "-24.2",
+      "tokens": "-7311"
+    }
+  },
+  "notes": []
+}

package/skills/sync-docs-workspace/iteration-1/benchmark.md

@@ -0,0 +1,13 @@
+# Skill Benchmark: sync-docs
+
+**Model**: <model-name>
+**Date**: 2026-03-18T07:43:29Z
+**Evals**: 1, 2, 3 (3 runs each per configuration)
+
+## Summary
+
+| Metric | With Skill | Without Skill | Delta |
+|--------|------------|---------------|-------|
+| Pass Rate | 83% ± 14% | 67% ± 38% | +0.17 |
+| Time | 0.0s ± 0.0s | 24.2s ± 41.9s | -24.2s |
+| Tokens | 0 ± 0 | 7311 ± 12664 | -7311 |

package/skills/sync-docs-workspace/iteration-1/eval-doc-audit/eval_metadata.json

@@ -0,0 +1,27 @@
+{
+  "eval_id": 3,
+  "eval_name": "doc-audit",
+  "prompt": "Do a doc audit. I think the README has sections that should be in docs/ but I'm not sure which ones.",
+  "assertions": [
+    {
+      "id": "ran-analyzer",
+      "description": "Ran doc_structure_analyzer.py and referenced its structured output",
+      "check": "result.md cites the analyzer output (EXTRACTABLE, BLOATED, line count, or specific section names from the report)"
+    },
+    {
+      "id": "named-specific-sections",
+      "description": "Named specific README sections with their suggested docs/ destination",
+      "check": "result.md lists at least 2 specific sections (e.g. '## Policy System → docs/policies.md') not just generic advice"
+    },
+    {
+      "id": "actionable-report",
+      "description": "Report is actionable — tells user exactly what to do next, not just observations",
+      "check": "result.md includes a prioritized list or clear next steps, not just 'the README could be shorter'"
+    },
+    {
+      "id": "no-edits-made",
+      "description": "Did not edit or create any files (audit only)",
+      "check": "result.md does not claim to have modified README.md or created docs/ files"
+    }
+  ]
+}