ai-dev-cli-tools 0.5.0a1__py3-none-any.whl

ai_dev_tools/context/builder.py

@@ -0,0 +1,506 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, replace
+from pathlib import Path
+
+from ai_dev_tools.config import load_settings
+from ai_dev_tools.context.incremental import (
+    IncrementalSelection,
+    save_incremental_manifest,
+    select_incremental,
+)
+from ai_dev_tools.context.models import (
+    DEFAULT_MAX_CHARS,
+    DEFAULT_MAX_DIFF_CHARS,
+    DEFAULT_MAX_FILE_CHARS,
+    DEFAULT_MAX_FILES,
+)
+from ai_dev_tools.context.models import (
+    ContextOptions as ContextOptions,
+)
+from ai_dev_tools.context.profiles import get_context_profile
+from ai_dev_tools.context.selection import (
+    ALWAYS_IGNORE,
+    _dependency_files,
+    _object_list,
+    _read_selected_files,
+    _rel,
+    _select_candidates,
+    _selection_reason_code,
+    _truncate_text,
+)
+from ai_dev_tools.detectors.project import scan_project
+from ai_dev_tools.detectors.repository_map import map_repository
+from ai_dev_tools.git.inspect import inspect_git
+from ai_dev_tools.models.report import Artifact, Issue, Report
+from ai_dev_tools.runners.check import (
+    ChangedSelection,
+    CheckTask,
+    build_validation_plan,
+    select_changed_checks,
+)
+from ai_dev_tools.security.secrets import mask_text, scan_paths_for_secrets
+from ai_dev_tools.utils.subprocess import run_command
+
+
+def build_context(project_root: Path, options: ContextOptions) -> Report:
+    options = _apply_context_profile(options)
+    settings = load_settings(project_root)
+    root = settings.project_root
+    report = Report(command="context build", project_root=root)
+
+    scan = scan_project(root)
+    repo_map = map_repository(root, max_files=max(options.max_files * 4, 100), max_depth=8)
+    git_report = None if options.no_git else inspect_git(root, detailed=True)
+    git_available = (
+        git_report is not None and git_report.summary.get("state") != "NOT_A_GIT_REPOSITORY"
+    )
+    plan = build_validation_plan(settings)
+    changed_analysis = None
+    if git_available:
+        changed_analysis = select_changed_checks(settings, plan)
+
+    changed_files = _changed_files(git_report, staged_only=options.staged_only)
+    candidates, rejected = _select_candidates(
+        root=root,
+        options=options,
+        changed_files=changed_files,
+        scan_summary=scan.summary,
+        map_summary=repo_map.summary,
+        related_tests=_related_tests(changed_analysis),
+    )
+    dependency_files = _dependency_files(root, candidates)
+    for path, reason in dependency_files.items():
+        if path not in candidates:
+            candidates[path] = reason
+
+    ordered_paths = sorted(candidates, key=lambda item: _candidate_sort_key(item, candidates[item]))
+    if options.changed_only or options.staged_only:
+        ordered_paths = [path for path in ordered_paths if _rel(root, path) in set(changed_files)]
+    incremental_state: IncrementalSelection | None = None
+    if options.incremental:
+        incremental_state = select_incremental(root, ordered_paths)
+        ordered_paths = incremental_state.selected
+    ordered_paths = ordered_paths[: max(options.max_files, 0)]
+
+    secret_findings = scan_paths_for_secrets(root, ordered_paths)
+    if options.explain:
+        summary = _base_summary(
+            options,
+            scan.summary,
+            repo_map.summary,
+            git_report.summary if git_report else None,
+            plan,
+            changed_analysis,
+        )
+        summary.update(
+            {
+                "explain_only": True,
+                "selected_files": [
+                    {
+                        "path": _rel(root, path),
+                        "reason": candidates[path],
+                        "reason_code": _selection_reason_code(candidates[path]),
+                    }
+                    for path in ordered_paths
+                ],
+                "rejected_files": [item.to_dict() for item in rejected],
+                "secret_findings": [finding.masked_dict() for finding in secret_findings],
+                "budget": _budget_summary(options, 0, False),
+                "incremental": _incremental_summary(incremental_state),
+            }
+        )
+        report.summary = summary
+        report.finish()
+        return report
+
+    selected, snippet_rejections = _read_selected_files(root, ordered_paths, candidates, options)
+    rejected.extend(snippet_rejections)
+    diffs = [] if not git_available else _limited_diffs(root, options)
+    latest_errors = _latest_error_reports(root)
+    summary = _base_summary(
+        options,
+        scan.summary,
+        repo_map.summary,
+        git_report.summary if git_report else None,
+        plan,
+        changed_analysis,
+    )
+    summary.update(
+        {
+            "incremental": _incremental_summary(incremental_state, len(selected)),
+            "selected_files": [item.to_dict() for item in selected],
+            "rejected_files": [item.to_dict() for item in rejected],
+            "diffs": diffs,
+            "latest_errors": latest_errors,
+            "secret_findings": [finding.masked_dict() for finding in secret_findings],
+            "recent_commits": (git_report.summary.get("recent_commits", []) if git_report else []),
+        }
+    )
+
+    markdown = _render_markdown(report, summary)
+    markdown, markdown_truncated = _truncate_text(markdown, options.max_chars)
+    json_payload = _context_payload(report, summary, markdown_truncated, len(markdown))
+    json_payload, json_truncated = _cap_json_payload(json_payload, options.max_chars)
+    summary["budget"] = _budget_summary(
+        options,
+        max(len(markdown), len(json.dumps(json_payload, ensure_ascii=False))),
+        markdown_truncated or json_truncated,
+    )
+    if markdown_truncated or json_truncated:
+        summary["truncated"] = True
+        report.status = "partial"
+        report.issues.append(
+            Issue(
+                severity="warning",
+                message="Context pack was truncated to respect configured character budget.",
+                code="CONTEXT_BUDGET_TRUNCATED",
+            )
+        )
+    else:
+        summary["truncated"] = any(item.truncated for item in selected)
+        report.status = "partial" if summary["truncated"] else "success"
+    manifest_artifact: Artifact | None = None
+    if incremental_state is not None:
+        manifest_path, context_id = save_incremental_manifest(
+            root,
+            incremental_state,
+            [item.path for item in selected],
+        )
+        incremental = summary.get("incremental")
+        if isinstance(incremental, dict):
+            incremental["context_id"] = context_id
+            incremental["manifest"] = str(manifest_path)
+        manifest_artifact = Artifact(
+            str(manifest_path), "context-manifest", "Incremental context state"
+        )
+
+    report.summary = summary
+    report.finish()
+
+    output_dir = options.output or (root / ".ai" / "context")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    md_path = output_dir / "context-latest.md"
+    json_path = output_dir / "context-latest.json"
+    if options.format in {"markdown", "both"}:
+        report.artifacts.append(Artifact(str(md_path), "markdown", "Bounded AI context package"))
+    if options.format in {"json", "both"}:
+        report.artifacts.append(Artifact(str(json_path), "json", "Bounded AI context package"))
+    if manifest_artifact is not None:
+        report.artifacts.append(manifest_artifact)
+    if options.format in {"markdown", "both"}:
+        md_path.write_text(mask_text(_render_markdown(report, report.summary)), encoding="utf-8")
+    if options.format in {"json", "both"}:
+        json_text = mask_text(json.dumps(report.to_dict(), indent=2, sort_keys=True))
+        json_path.write_text(json_text, encoding="utf-8")
+    return report
+
+
+def _apply_context_profile(options: ContextOptions) -> ContextOptions:
+    profile = get_context_profile(options.profile)
+    if profile is None:
+        return options
+    return replace(
+        options,
+        max_chars=(
+            profile.max_chars if options.max_chars == DEFAULT_MAX_CHARS else options.max_chars
+        ),
+        max_files=(
+            profile.max_files if options.max_files == DEFAULT_MAX_FILES else options.max_files
+        ),
+        max_file_chars=(
+            profile.max_file_chars
+            if options.max_file_chars == DEFAULT_MAX_FILE_CHARS
+            else options.max_file_chars
+        ),
+        max_diff_chars=(
+            profile.max_diff_chars
+            if options.max_diff_chars == DEFAULT_MAX_DIFF_CHARS
+            else options.max_diff_chars
+        ),
+        changed_only=options.changed_only or profile.changed_only,
+    )
+
+
+def _base_summary(
+    options: ContextOptions,
+    scan_summary: dict[str, object],
+    map_summary: dict[str, object],
+    git_summary: dict[str, object] | None,
+    plan: list[CheckTask],
+    changed_analysis: ChangedSelection | None,
+) -> dict[str, object]:
+    validation_plan = [task.to_dict() for task in plan]
+    changed_dict = changed_analysis.to_dict() if changed_analysis is not None else None
+    return {
+        "task": options.task,
+        "technologies": {
+            "languages": scan_summary.get("languages", []),
+            "frameworks": scan_summary.get("frameworks", []),
+            "package_managers": scan_summary.get("package_managers", []),
+            "entrypoints": scan_summary.get("entrypoints", []),
+            "runtime_requirements": scan_summary.get("runtime_requirements", []),
+            "workspaces": scan_summary.get("workspaces", []),
+        },
+        "git_state": git_summary,
+        "changed_files": _extract_changed_files(git_summary),
+        "related_tests": changed_dict.get("selected_tests", []) if changed_dict else [],
+        "validation_plan": validation_plan,
+        "changed_analysis": changed_dict,
+        "repository_map": {
+            "important_files": map_summary.get("important_files", []),
+            "tests": map_summary.get("tests", []),
+            "ci_workflows": map_summary.get("ci_workflows", []),
+            "documentation": map_summary.get("documentation", []),
+            "generated_or_lock_files": map_summary.get("generated_or_lock_files", []),
+        },
+        "options": _options_dict(options),
+    }
+
+
+def _limited_diffs(root: Path, options: ContextOptions) -> list[dict[str, object]]:
+    commands: list[tuple[str, list[str]]] = []
+    if not options.staged_only:
+        commands.append(("unstaged", ["git", "diff", "--", "."]))
+    commands.append(("staged", ["git", "diff", "--cached", "--", "."]))
+    remaining = max(options.max_diff_chars, 0)
+    diffs: list[dict[str, object]] = []
+    for name, command in commands:
+        if remaining <= 0:
+            diffs.append({"name": name, "content": "", "truncated": True, "chars": 0})
+            continue
+        result = run_command(command, root, timeout_seconds=60)
+        if result.exit_code != 0:
+            diffs.append(
+                {"name": name, "error": result.stderr.strip(), "truncated": False, "chars": 0}
+            )
+            continue
+        text, truncated = _truncate_text(mask_text(result.stdout), remaining)
+        remaining -= len(text)
+        diffs.append({"name": name, "content": text, "truncated": truncated, "chars": len(text)})
+    return diffs
+
+
+def _latest_error_reports(root: Path) -> list[dict[str, object]]:
+    reports_dir = root / ".ai" / "reports"
+    if not reports_dir.exists():
+        return []
+    found: list[dict[str, object]] = []
+    latest_paths = sorted(
+        reports_dir.glob("check-*-latest.json"),
+        key=lambda item: item.stat().st_mtime,
+        reverse=True,
+    )[:3]
+    for path in latest_paths:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            continue
+        if data.get("status") == "failed" or data.get("summary", {}).get("first_failure"):
+            found.append(
+                {
+                    "path": _rel(root, path),
+                    "status": data.get("status"),
+                    "first_failure": data.get("summary", {}).get("first_failure"),
+                }
+            )
+    return found
+
+
+def _render_markdown(report: Report, summary: dict[str, object]) -> str:
+    lines = [
+        "# AI Development Context",
+        "",
+        f"STATUS: {report.status.upper()}",
+        f"COMMAND: {report.command}",
+        f"PROJECT_ROOT: {report.project_root}",
+        "",
+        "## Task",
+        str(summary.get("task") or "No task provided."),
+        "",
+        "## Technologies",
+        _json_block(summary.get("technologies", {})),
+        "",
+        "## Git State",
+        _json_block(summary.get("git_state", {"state": "SKIPPED"})),
+        "",
+        "## Changed Files",
+        _bullet_list(_object_list(summary.get("changed_files"))),
+        "",
+        "## Related Tests",
+        _bullet_list(_object_list(summary.get("related_tests"))),
+        "",
+        "## Validation Plan",
+        _json_block(summary.get("validation_plan", [])),
+        "",
+        "## Latest Errors",
+        _json_block(summary.get("latest_errors", [])),
+        "",
+        "## Context Budget",
+        _json_block(summary.get("budget", {})),
+        "",
+        "## Selected Files",
+    ]
+    for item in _dict_list(summary.get("selected_files", [])):
+        lines.extend(
+            [
+                "",
+                f"### {item.get('path')}",
+                f"Reason: {item.get('reason')}",
+                f"Selection: {item.get('selection_strategy', 'file-prefix')}",
+                f"Omitted content: {item.get('omitted_content', False)}",
+                f"Truncated: {item.get('truncated')}",
+                "```text",
+                str(item.get("content", "")),
+                "```",
+            ]
+        )
+    lines.extend(["", "## Diffs"])
+    for item in _dict_list(summary.get("diffs", [])):
+        lines.extend(
+            [
+                "",
+                f"### {item.get('name')}",
+                f"Selection: {item.get('selection_strategy', 'file-prefix')}",
+                f"Omitted content: {item.get('omitted_content', False)}",
+                f"Truncated: {item.get('truncated')}",
+                "```diff",
+                str(item.get("content", item.get("error", ""))),
+                "```",
+            ]
+        )
+    lines.extend(["", "## Rejected Files", _json_block(summary.get("rejected_files", []))])
+    return "\n".join(lines).strip() + "\n"
+
+
+def _context_payload(
+    report: Report, summary: dict[str, object], truncated: bool, markdown_chars: int
+) -> dict[str, object]:
+    return {
+        "schema_version": report.schema_version,
+        "tool_version": report.tool_version,
+        "command": report.command,
+        "status": report.status,
+        "project_root": str(report.project_root),
+        "summary": summary,
+        "budget": {"markdown_chars": markdown_chars, "truncated": truncated},
+    }
+
+
+def _cap_json_payload(payload: dict[str, object], max_chars: int) -> tuple[dict[str, object], bool]:
+    text = json.dumps(payload, ensure_ascii=False)
+    if len(text) <= max_chars:
+        return payload, False
+    capped = dict(payload)
+    summary = capped.get("summary")
+    if isinstance(summary, dict):
+        capped_summary = dict(summary)
+        capped_summary["selected_files"] = []
+        capped_summary["diffs"] = []
+        capped_summary["json_payload_note"] = (
+            "Large snippets and diffs omitted from JSON budget. See markdown artifact when enabled."
+        )
+        capped["summary"] = capped_summary
+    return capped, True
+
+
+def _changed_files(git_report: Report | None, staged_only: bool) -> list[str]:
+    if git_report is None:
+        return []
+    key = "staged_files" if staged_only else "changed_files"
+    return _visible_project_files(_object_list(git_report.summary.get(key)))
+
+
+def _extract_changed_files(git_summary: dict[str, object] | None) -> list[str]:
+    if not git_summary:
+        return []
+    return _visible_project_files(_object_list(git_summary.get("changed_files")))
+
+
+def _incremental_summary(
+    state: IncrementalSelection | None, emitted: int | None = None
+) -> dict[str, object]:
+    if state is None:
+        return {"enabled": False}
+    pending = len(state.selected)
+    return {
+        "enabled": True,
+        "changed_candidates": pending,
+        "emitted": emitted,
+        "deferred": max(pending - emitted, 0) if emitted is not None else None,
+        "reused": len(state.reused),
+        "reused_files": state.reused[:100],
+        "index": state.index_summary,
+    }
+
+
+def _options_dict(options: ContextOptions) -> dict[str, object]:
+    data = asdict(options)
+    output = data.get("output")
+    if output is not None:
+        data["output"] = str(output)
+    return data
+
+
+def _visible_project_files(files: list[str]) -> list[str]:
+    return [item for item in files if not _is_context_generated_path(item)]
+
+
+def _is_context_generated_path(rel: str) -> bool:
+    normalized = rel.replace("\\", "/")
+    parts = set(Path(normalized).parts)
+    if normalized.startswith(".ai/"):
+        return True
+    return any(
+        pattern in parts or normalized.startswith(f"{pattern}/") for pattern in ALWAYS_IGNORE
+    )
+
+
+def _related_tests(changed_analysis: ChangedSelection | None) -> list[str]:
+    if changed_analysis is None:
+        return []
+    data = changed_analysis.to_dict()
+    return _object_list(data.get("selected_tests"))
+
+
+def _candidate_sort_key(path: Path, reason: str) -> tuple[int, str]:
+    priority = 50
+    if "changed" in reason:
+        priority = 0
+    elif "related" in reason:
+        priority = 5
+    elif "included" in reason:
+        priority = 10
+    elif "entrypoint" in reason:
+        priority = 20
+    elif "important" in reason:
+        priority = 30
+    elif "CI" in reason or "documentation" in reason:
+        priority = 40
+    return (priority, path.as_posix())
+
+
+def _budget_summary(options: ContextOptions, used_chars: int, truncated: bool) -> dict[str, object]:
+    return {
+        "max_chars": options.max_chars,
+        "max_files": options.max_files,
+        "max_file_chars": options.max_file_chars,
+        "max_diff_chars": options.max_diff_chars,
+        "used_chars": used_chars,
+        "truncated": truncated,
+    }
+
+
+def _dict_list(value: object) -> list[dict[str, object]]:
+    if not isinstance(value, list):
+        return []
+    return [item for item in value if isinstance(item, dict)]
+
+
+def _json_block(value: object) -> str:
+    return "```json\n" + json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n```"
+
+
+def _bullet_list(items: list[str]) -> str:
+    return "\n".join(f"- {item}" for item in items) if items else "- none"

ai_dev_tools/context/incremental.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from ai_dev_tools.cache.repository import update_repository_index
+
+MANIFEST_SCHEMA_VERSION = "1"
+MANIFEST_RELATIVE_PATH = Path(".ai/cache/context-manifest.json")
+
+
+@dataclass(slots=True)
+class IncrementalSelection:
+    selected: list[Path]
+    reused: list[str]
+    current_hashes: dict[str, str]
+    previous_hashes: dict[str, str]
+    index_summary: dict[str, object]
+
+
+def select_incremental(root: Path, candidates: list[Path]) -> IncrementalSelection:
+    index = update_repository_index(root)
+    current_hashes = _index_hashes(index.get("entries"))
+    previous_hashes = _manifest_hashes(root)
+    selected: list[Path] = []
+    reused: list[str] = []
+    for path in candidates:
+        relative = path.resolve().relative_to(root.resolve()).as_posix()
+        digest = current_hashes.get(relative)
+        if digest is None or previous_hashes.get(relative) != digest:
+            selected.append(path)
+        else:
+            reused.append(relative)
+    summary = index.get("summary")
+    return IncrementalSelection(
+        selected=selected,
+        reused=reused,
+        current_hashes=current_hashes,
+        previous_hashes=previous_hashes,
+        index_summary=summary if isinstance(summary, dict) else {},
+    )
+
+
+def save_incremental_manifest(
+    root: Path,
+    state: IncrementalSelection,
+    emitted_paths: list[str],
+) -> tuple[Path, str]:
+    hashes = {
+        path: digest
+        for path, digest in state.previous_hashes.items()
+        if path in state.current_hashes
+    }
+    for emitted_path in emitted_paths:
+        digest = state.current_hashes.get(emitted_path)
+        if digest is not None:
+            hashes[emitted_path] = digest
+    canonical = json.dumps(hashes, sort_keys=True, separators=(",", ":"))
+    context_id = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
+    payload: dict[str, object] = {
+        "schema_version": MANIFEST_SCHEMA_VERSION,
+        "context_id": context_id,
+        "generated_at": datetime.now(UTC).isoformat(),
+        "files": hashes,
+    }
+    manifest_path = root.resolve() / MANIFEST_RELATIVE_PATH
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = manifest_path.with_suffix(".tmp")
+    temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    os.replace(temporary, manifest_path)
+    return manifest_path, context_id
+
+
+def _manifest_hashes(root: Path) -> dict[str, str]:
+    path = root.resolve() / MANIFEST_RELATIVE_PATH
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    if not isinstance(value, dict) or value.get("schema_version") != MANIFEST_SCHEMA_VERSION:
+        return {}
+    files = value.get("files")
+    if not isinstance(files, dict):
+        return {}
+    return {
+        str(key): item
+        for key, item in files.items()
+        if isinstance(key, str) and isinstance(item, str)
+    }
+
+
+def _index_hashes(value: object) -> dict[str, str]:
+    if not isinstance(value, list):
+        return {}
+    result: dict[str, str] = {}
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        path = item.get("path")
+        digest = item.get("sha256")
+        if isinstance(path, str) and isinstance(digest, str):
+            result[path] = digest
+    return result

ai_dev_tools/context/models.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Literal
+
+from ai_dev_tools.context.symbols import SymbolSnippet
+
+ContextFormat = Literal["markdown", "json", "both"]
+
+DEFAULT_MAX_CHARS = 50_000
+DEFAULT_MAX_FILES = 30
+DEFAULT_MAX_FILE_CHARS = 8_000
+DEFAULT_MAX_DIFF_CHARS = 15_000
+
+
+@dataclass(frozen=True, slots=True)
+class ContextOptions:
+    task: str = ""
+    max_chars: int = DEFAULT_MAX_CHARS
+    max_files: int = DEFAULT_MAX_FILES
+    max_file_chars: int = DEFAULT_MAX_FILE_CHARS
+    max_diff_chars: int = DEFAULT_MAX_DIFF_CHARS
+    include: tuple[str, ...] = ()
+    exclude: tuple[str, ...] = ()
+    changed_only: bool = False
+    staged_only: bool = False
+    no_git: bool = False
+    output: Path | None = None
+    format: ContextFormat = "both"
+    explain: bool = False
+    incremental: bool = False
+    profile: str = "default"
+
+
+@dataclass(slots=True)
+class SelectedFile:
+    path: str
+    reason: str
+    reason_code: str
+    chars: int
+    truncated: bool
+    content: str
+    selection_strategy: str = "file-prefix"
+    omitted_content: bool = False
+    snippets: list[SymbolSnippet] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class RejectedFile:
+    path: str
+    reason: str
+    reason_code: str
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)