agentic-project-kit 1.0.1__py3-none-any.whl

agentic_project_kit/removed_ns_commands.py

@@ -0,0 +1,458 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable, Sequence
+
+
+TEXT_SUFFIXES = {
+    "",
+    ".py",
+    ".sh",
+    ".bash",
+    ".zsh",
+    ".md",
+    ".rst",
+    ".txt",
+    ".toml",
+    ".yaml",
+    ".yml",
+    ".json",
+    ".ini",
+    ".cfg",
+    ".conf",
+    ".jinja",
+    ".j2",
+}
+
+BINARY_SUFFIXES = {
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".pdf",
+    ".zip",
+    ".gz",
+    ".tgz",
+    ".pyc",
+    ".sqlite",
+    ".db",
+    ".woff",
+    ".woff2",
+}
+
+SEARCH_ROOTS = ("ns", "ns-menu", "tools", "src", "tests", "docs", ".agentic", "scripts", "bin")
+
+EXCLUDED_PATH_PREFIXES = (
+    "docs/reports/",
+    "docs/reports_archive/",
+    ".agentic/transfer/inbox/",
+    ".agentic/transfer/outbox/",
+)
+
+EXCLUDED_PATHS = {
+    ".agentic/transfer/inbox/current.yaml",
+    ".agentic/transfer/inbox/next_command.py.txt",
+    ".agentic/transfer/outbox/last_result.txt",
+}
+
+NS_PATTERNS = (
+    r"\bns-dev\b",
+    r"\bns\s+dev\b",
+    r"\bns-go\b",
+    r"\bns\s+go\b",
+    r"\bns-[a-z0-9][a-z0-9_.-]*\b",
+    r"\bns\s+[a-z0-9][a-z0-9_.-]*(?:\s+[a-z0-9][a-z0-9_.-]*)?\b",
+)
+
+
+@dataclass(frozen=True)
+class GitResult:
+    argv: list[str]
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+@dataclass(frozen=True)
+class Hit:
+    ref: str
+    path: str
+    line_no: int
+    line: str
+
+
+@dataclass(frozen=True)
+class CommandCandidate:
+    name: str
+    evidence_count_old: int
+    evidence_count_new: int
+    evidence_count_main: int
+    removed_or_decreased_signal: bool
+
+
+def _git(args: Sequence[str], *, check: bool = True) -> GitResult:
+    argv = ["git", *args]
+    cp = subprocess.run(
+        argv,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )
+    result = GitResult(argv=argv, returncode=cp.returncode, stdout=cp.stdout, stderr=cp.stderr)
+    if check and cp.returncode != 0:
+        raise RuntimeError(
+            "git command failed\n"
+            f"argv={argv!r}\n"
+            f"returncode={cp.returncode}\n"
+            f"stdout={cp.stdout}\n"
+            f"stderr={cp.stderr}"
+        )
+    return result
+
+
+def _resolve_ref(ref: str) -> str:
+    candidates = [ref]
+    if not ref.startswith("v"):
+        candidates.append(f"v{ref}")
+    elif ref.startswith("v"):
+        candidates.append(ref[1:])
+
+    for candidate in dict.fromkeys(candidates):
+        result = _git(["rev-parse", "--verify", "--quiet", candidate], check=False)
+        if result.returncode == 0:
+            return candidate
+
+    tags = _git(["tag", "-l", f"*{ref.lstrip('v')}*"], check=False).stdout.strip()
+    raise RuntimeError(f"Could not resolve release ref {ref!r}. Matching tags: {tags}")
+
+
+def _list_files(ref: str) -> list[str]:
+    result = _git(["ls-tree", "-r", "--name-only", ref])
+    return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+
+
+def _path_in_scope(path: str) -> bool:
+    normalized = path.strip("/")
+    if normalized in EXCLUDED_PATHS:
+        return False
+    if any(normalized.startswith(prefix) for prefix in EXCLUDED_PATH_PREFIXES):
+        return False
+    return any(normalized == root or normalized.startswith(root + "/") for root in SEARCH_ROOTS)
+
+
+def _is_probably_text(path: str) -> bool:
+    if not _path_in_scope(path):
+        return False
+    suffix = Path(path).suffix.lower()
+    if suffix in BINARY_SUFFIXES:
+        return False
+    if suffix in TEXT_SUFFIXES:
+        return True
+    return Path(path).name in {"ns", "ns-menu"}
+
+
+def _file_text(ref: str, path: str) -> str | None:
+    if not _is_probably_text(path):
+        return None
+    result = _git(["show", f"{ref}:{path}"], check=False)
+    if result.returncode != 0:
+        return None
+    return result.stdout
+
+
+def _collect_ref_hits(ref: str) -> list[Hit]:
+    regexes = [re.compile(pattern, re.IGNORECASE) for pattern in NS_PATTERNS]
+    hits: list[Hit] = []
+    for path in _list_files(ref):
+        text = _file_text(ref, path)
+        if text is None:
+            continue
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            stripped = line.strip()
+            if stripped and any(regex.search(stripped) for regex in regexes):
+                hits.append(Hit(ref=ref, path=path, line_no=line_no, line=stripped[:320]))
+    return hits
+
+
+def _iter_main_text_paths() -> Iterable[Path]:
+    for root_name in SEARCH_ROOTS:
+        root = Path(root_name)
+        if not root.exists():
+            continue
+        if root.is_file():
+            if _is_probably_text(str(root)):
+                yield root
+            continue
+        for path in root.rglob("*"):
+            if path.is_file() and _is_probably_text(str(path)):
+                yield path
+
+
+def _collect_main_hits() -> list[Hit]:
+    regexes = [re.compile(pattern, re.IGNORECASE) for pattern in NS_PATTERNS]
+    hits: list[Hit] = []
+    for path in _iter_main_text_paths():
+        try:
+            text = path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            continue
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            stripped = line.strip()
+            if stripped and any(regex.search(stripped) for regex in regexes):
+                hits.append(Hit(ref="main-worktree", path=str(path), line_no=line_no, line=stripped[:320]))
+    return hits
+
+
+def _strip_shell_suffix(name: str) -> str:
+    for suffix in (".sh", ".bash", ".zsh", ".py"):
+        if name.endswith(suffix):
+            return name[: -len(suffix)]
+    return name
+
+
+def _normalize_command_names(hits: Iterable[Hit]) -> set[str]:
+    names: set[str] = set()
+    for hit in hits:
+        line = hit.line.lower()
+        for match in re.finditer(r"\bns-[a-z0-9][a-z0-9_.-]*\b", line):
+            names.add(_strip_shell_suffix(match.group(0)))
+
+        for match in re.finditer(r"\bns\s+[a-z0-9][a-z0-9_.-]*(?:\s+[a-z0-9][a-z0-9_.-]*)?\b", line):
+            normalized = " ".join(match.group(0).split())
+            names.add(normalized)
+            parts = normalized.split()
+            if len(parts) >= 2:
+                names.add(" ".join(parts[:2]))
+    return names
+
+
+def _count_name(hits: Iterable[Hit], name: str) -> int:
+    if " " in name:
+        pattern = r"\b" + re.escape(name).replace(r"\ ", r"\s+") + r"\b"
+    else:
+        pattern = r"\b" + re.escape(name) + r"\b"
+    regex = re.compile(pattern, re.IGNORECASE)
+    return sum(1 for hit in hits if regex.search(hit.line))
+
+
+def _removed_lines(old_ref: str, new_ref: str) -> list[str]:
+    result = _git(
+        [
+            "diff",
+            "--unified=0",
+            f"{old_ref}..{new_ref}",
+            "--",
+            *SEARCH_ROOTS,
+        ],
+        check=False,
+    )
+    regexes = [re.compile(pattern, re.IGNORECASE) for pattern in NS_PATTERNS]
+    lines: list[str] = []
+    current_path: str | None = None
+
+    for line in result.stdout.splitlines():
+        if line.startswith("diff --git "):
+            parts = line.split()
+            if len(parts) >= 4:
+                current_path = parts[2][2:] if parts[2].startswith("a/") else parts[2]
+            else:
+                current_path = None
+            continue
+
+        if current_path is not None and not _path_in_scope(current_path):
+            continue
+
+        if not line.startswith("-") or line.startswith("---"):
+            continue
+        if any(regex.search(line) for regex in regexes):
+            lines.append(line[:360])
+    return lines
+
+
+def _deleted_files(old_ref: str, new_ref: str) -> list[str]:
+    result = _git(
+        [
+            "diff",
+            "--name-status",
+            "--diff-filter=D",
+            f"{old_ref}..{new_ref}",
+            "--",
+            *SEARCH_ROOTS,
+        ],
+        check=False,
+    )
+    filtered: list[str] = []
+    for line in result.stdout.splitlines():
+        if not line.strip():
+            continue
+        parts = line.split(maxsplit=1)
+        if len(parts) != 2:
+            continue
+        status, deleted_path = parts
+        if status == "D" and _path_in_scope(deleted_path):
+            filtered.append(line)
+    return filtered
+
+
+def build_removed_ns_command_report(old: str, new: str) -> dict:
+    old_ref = _resolve_ref(old)
+    new_ref = _resolve_ref(new)
+
+    old_hits = _collect_ref_hits(old_ref)
+    new_hits = _collect_ref_hits(new_ref)
+    main_hits = _collect_main_hits()
+    removed = _removed_lines(old_ref, new_ref)
+
+    removed_hit_proxy = [Hit(ref="diff", path="diff", line_no=i + 1, line=line) for i, line in enumerate(removed)]
+
+    names = sorted(
+        _normalize_command_names(old_hits)
+        | _normalize_command_names(new_hits)
+        | _normalize_command_names(main_hits)
+        | _normalize_command_names(removed_hit_proxy)
+    )
+
+    candidates: list[CommandCandidate] = []
+    for name in names:
+        old_count = _count_name(old_hits, name)
+        new_count = _count_name(new_hits, name)
+        main_count = _count_name(main_hits, name)
+        removed_count = _count_name(removed_hit_proxy, name)
+        candidates.append(
+            CommandCandidate(
+                name=name,
+                evidence_count_old=old_count,
+                evidence_count_new=new_count,
+                evidence_count_main=main_count,
+                removed_or_decreased_signal=removed_count > 0 or old_count > new_count,
+            )
+        )
+
+    candidates.sort(
+        key=lambda item: (
+            not item.removed_or_decreased_signal,
+            -item.evidence_count_old,
+            item.name,
+        )
+    )
+
+    return {
+        "schema_version": 1,
+        "kind": "removed_ns_command_diagnosis",
+        "created_at_utc": datetime.now(timezone.utc).isoformat(),
+        "old_ref": old_ref,
+        "new_ref": new_ref,
+        "old_commit": _git(["rev-parse", old_ref]).stdout.strip(),
+        "new_commit": _git(["rev-parse", new_ref]).stdout.strip(),
+        "candidate_commands": [asdict(candidate) for candidate in candidates],
+        "deleted_files": _deleted_files(old_ref, new_ref),
+        "removed_lines": removed,
+        "old_hits": [asdict(hit) for hit in old_hits],
+        "new_hits": [asdict(hit) for hit in new_hits],
+        "main_hits": [asdict(hit) for hit in main_hits],
+    }
+
+
+def write_markdown_report(report: dict, path: Path) -> None:
+    lines: list[str] = []
+    lines.append("# Removed ns command diagnosis")
+    lines.append("")
+    lines.append(f"- old_ref: `{report['old_ref']}` / `{report['old_commit']}`")
+    lines.append(f"- new_ref: `{report['new_ref']}` / `{report['new_commit']}`")
+    lines.append("")
+    lines.append("## Candidate commands")
+    lines.append("")
+    lines.append("| command | old hits | new hits | main hits | removed/decreased |")
+    lines.append("|---|---:|---:|---:|---|")
+    for item in report["candidate_commands"]:
+        if item["removed_or_decreased_signal"] or item["name"] in {"ns-dev", "ns dev", "ns-go", "ns go"}:
+            lines.append(
+                f"| `{item['name']}` | {item['evidence_count_old']} | "
+                f"{item['evidence_count_new']} | {item['evidence_count_main']} | "
+                f"{item['removed_or_decreased_signal']} |"
+            )
+    lines.append("")
+    lines.append("## Deleted files")
+    lines.append("")
+    for line in report["deleted_files"][:300]:
+        lines.append(f"- `{line}`")
+    if not report["deleted_files"]:
+        lines.append("- none")
+    lines.append("")
+    lines.append("## Removed ns lines")
+    lines.append("")
+    for line in report["removed_lines"][:300]:
+        lines.append(f"```text\n{line}\n```")
+    if not report["removed_lines"]:
+        lines.append("- none")
+    lines.append("")
+    lines.append("## Old release hits")
+    lines.append("")
+    for hit in report["old_hits"][:500]:
+        lines.append(f"- `{hit['path']}:{hit['line_no']}`: `{hit['line']}`")
+    lines.append("")
+    lines.append("## New release hits")
+    lines.append("")
+    for hit in report["new_hits"][:300]:
+        lines.append(f"- `{hit['path']}:{hit['line_no']}`: `{hit['line']}`")
+    lines.append("")
+    lines.append("## Current main hits")
+    lines.append("")
+    for hit in report["main_hits"][:300]:
+        lines.append(f"- `{hit['path']}:{hit['line_no']}`: `{hit['line']}`")
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Diagnose ns commands removed or reduced between release refs.")
+    parser.add_argument("--old", default="0.4.6")
+    parser.add_argument("--new", default="0.4.8")
+    parser.add_argument("--json-out", default=None)
+    parser.add_argument("--md-out", default=None)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args(argv)
+
+    report = build_removed_ns_command_report(old=args.old, new=args.new)
+
+    if args.json_out:
+        Path(args.json_out).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    if args.md_out:
+        write_markdown_report(report, Path(args.md_out))
+
+    if args.json:
+        print(json.dumps(report, indent=2, ensure_ascii=False))
+    else:
+        print("REMOVED_NS_COMMAND_DIAGNOSIS")
+        print(f"old_ref={report['old_ref']}")
+        print(f"new_ref={report['new_ref']}")
+        print(f"old_commit={report['old_commit']}")
+        print(f"new_commit={report['new_commit']}")
+        print("candidate_commands:")
+        for item in report["candidate_commands"][:40]:
+            print(
+                "- {name}: old={old} new={new} main={main} removed_or_decreased={signal}".format(
+                    name=item["name"],
+                    old=item["evidence_count_old"],
+                    new=item["evidence_count_new"],
+                    main=item["evidence_count_main"],
+                    signal=item["removed_or_decreased_signal"],
+                )
+            )
+        if args.json_out:
+            print(f"json_out={args.json_out}")
+        if args.md_out:
+            print(f"md_out={args.md_out}")
+        print("RESULT=REMOVED_NS_COMMAND_DIAGNOSIS_DONE")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentic_project_kit/removed_source_audit.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+import re
+
+
+DEFAULT_CENTRAL_TARGET = "docs/planning/PROJECT_DIRECTION.yaml"
+DEFAULT_MANIFESTS = (
+    "docs/DOCUMENTATION_REGISTRY.yaml",
+    "docs/DOCUMENTATION_COVERAGE.yaml",
+    "docs/DOC_REGISTRY_SCOPE.yaml",
+)
+SCAN_SUFFIXES = {".md", ".py", ".yaml", ".yml"}
+SCAN_ROOTS = ("docs", "src", "tests")
+EXTRA_SCAN_FILES = ("README.md",)
+
+
+@dataclass(frozen=True)
+class RemovedSourceFinding:
+    path: str
+    exists: bool
+    registered_refs: list[str] = field(default_factory=list)
+    source_file_style_refs: list[str] = field(default_factory=list)
+    live_refs: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return (
+            not self.exists
+            and not self.registered_refs
+            and not self.source_file_style_refs
+            and not self.live_refs
+        )
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "path": self.path,
+            "exists": self.exists,
+            "registered_refs": self.registered_refs,
+            "source_file_style_refs": self.source_file_style_refs,
+            "live_refs": self.live_refs,
+            "ok": self.ok,
+        }
+
+
+@dataclass(frozen=True)
+class RemovedSourceAuditResult:
+    result_status: str
+    central_target: str
+    audited_count: int
+    findings: list[RemovedSourceFinding]
+
+    @property
+    def ok(self) -> bool:
+        return self.result_status == "PASS"
+
+    @property
+    def returncode(self) -> int:
+        return 0 if self.ok else 1
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "result_status": self.result_status,
+            "central_target": self.central_target,
+            "audited_count": self.audited_count,
+            "findings": [finding.as_dict() for finding in self.findings],
+        }
+
+
+def _rel(path: Path, root: Path) -> str:
+    return path.resolve().relative_to(root.resolve()).as_posix()
+
+
+def _iter_scan_files(root: Path) -> list[Path]:
+    files: list[Path] = []
+    for scan_root in SCAN_ROOTS:
+        base = root / scan_root
+        if not base.exists():
+            continue
+        for path in base.rglob("*"):
+            if not path.is_file():
+                continue
+            if "docs/reports" in path.as_posix():
+                continue
+            if path.suffix in SCAN_SUFFIXES:
+                files.append(path)
+    for extra in EXTRA_SCAN_FILES:
+        path = root / extra
+        if path.exists() and path.is_file():
+            files.append(path)
+    return sorted(set(files))
+
+
+def _line_refs(path: Path, root: Path, needle: str) -> list[str]:
+    refs: list[str] = []
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except UnicodeDecodeError:
+        return refs
+    rel = _rel(path, root)
+    for lineno, line in enumerate(lines, start=1):
+        if needle in line:
+            refs.append(f"{rel}:{lineno}:{line.strip()}")
+    return refs
+
+
+def _extract_removed_source_paths(root: Path, central_target: str) -> list[str]:
+    central_path = root / central_target
+    if not central_path.exists():
+        return []
+    text = central_path.read_text(encoding="utf-8")
+
+    # Intentionally dependency-free YAML-ish parser.  It recognizes both
+    # centralization forms used in PROJECT_DIRECTION.yaml:
+    #
+    #   removed_source:
+    #     status: removed_source
+    #     path: docs/old.md
+    #
+    #   removed_sources:
+    #     - path: docs/old-a.md
+    #       status: removed_source
+    #
+    # A candidate path is counted only when a nearby local block contains
+    # status: removed_source.  This keeps ordinary source_files entries out of
+    # the removed-source set while still supporting both key orders.
+    removed: list[str] = []
+    pattern = re.compile(
+        r"^\s*(?:-\s+)?path:\s+(docs/[^\s#]+)\s*$",
+        flags=re.MULTILINE,
+    )
+
+    for match in pattern.finditer(text):
+        candidate = match.group(1)
+        window_start = max(0, match.start() - 420)
+        window_end = min(len(text), match.end() + 420)
+        window = text[window_start:window_end]
+        if re.search(
+            r"(?m)^\s*status:\s*removed_source\s*$",
+            window,
+        ):
+            removed.append(candidate)
+
+    return sorted(set(removed))
+
+def _source_file_style_refs(root: Path, central_target: str, removed_path: str) -> list[str]:
+    path = root / central_target
+    if not path.exists():
+        return []
+    refs: list[str] = []
+    lines = path.read_text(encoding="utf-8").splitlines()
+    pattern = re.compile(rf"^\s*-\s+{re.escape(removed_path)}\s*$")
+    for lineno, line in enumerate(lines, start=1):
+        if pattern.match(line):
+            refs.append(f"{central_target}:{lineno}:{line.strip()}")
+    return refs
+
+
+def _manifest_refs(root: Path, removed_path: str, manifests: tuple[str, ...] = DEFAULT_MANIFESTS) -> list[str]:
+    refs: list[str] = []
+    for manifest in manifests:
+        path = root / manifest
+        if path.exists():
+            refs.extend(_line_refs(path, root, removed_path))
+    return refs
+
+
+def _live_refs(root: Path, central_target: str, removed_path: str) -> list[str]:
+    refs: list[str] = []
+    for path in _iter_scan_files(root):
+        rel = _rel(path, root)
+        if rel == central_target:
+            continue
+        if rel.startswith("docs/reports/"):
+            continue
+        refs.extend(_line_refs(path, root, removed_path))
+    return refs
+
+
+def build_removed_source_audit(
+    root: Path,
+    paths: list[str] | None = None,
+    central_target: str = DEFAULT_CENTRAL_TARGET,
+) -> RemovedSourceAuditResult:
+    root = root.resolve()
+    audit_paths = sorted(set(paths or _extract_removed_source_paths(root, central_target)))
+    findings = [
+        RemovedSourceFinding(
+            path=removed_path,
+            exists=(root / removed_path).exists(),
+            registered_refs=_manifest_refs(root, removed_path),
+            source_file_style_refs=_source_file_style_refs(root, central_target, removed_path),
+            live_refs=_live_refs(root, central_target, removed_path),
+        )
+        for removed_path in audit_paths
+    ]
+    status = "PASS" if all(finding.ok for finding in findings) else "FAIL"
+    return RemovedSourceAuditResult(
+        result_status=status,
+        central_target=central_target,
+        audited_count=len(findings),
+        findings=findings,
+    )
+
+
+def render_removed_source_audit(result: RemovedSourceAuditResult) -> str:
+    lines = [
+        "REMOVED_SOURCE_AUDIT",
+        f"STATUS={result.result_status}",
+        f"CENTRAL_TARGET={result.central_target}",
+        f"AUDITED={result.audited_count}",
+    ]
+    for finding in result.findings:
+        lines.append(f"- PATH={finding.path} OK={finding.ok}")
+        if finding.exists:
+            lines.append("  EXISTS=yes")
+        for label, refs in (
+            ("REGISTERED_REF", finding.registered_refs),
+            ("SOURCE_FILES_STYLE_REF", finding.source_file_style_refs),
+            ("LIVE_REF", finding.live_refs),
+        ):
+            for ref in refs:
+                lines.append(f"  {label} {ref}")
+    return "\n".join(lines) + "\n"

agentic_project_kit/rendering.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+from string import Template
+
+
+def render_template_string(template: str, context: dict[str, str]) -> str:
+    return Template(template).safe_substitute(context)
+
+
+def write_file(path: Path, content: str, overwrite: bool = False) -> None:
+    if path.exists() and not overwrite:
+        raise FileExistsError(f"Refusing to overwrite existing file: {path}")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content.lstrip(), encoding="utf-8")