algo-cli-runtime 0.14.0__py3-none-any.whl

algo_cli/intelligence/incremental_index.py

@@ -0,0 +1,316 @@
+"""Incremental Index — mtime-watermark based change tracking.
+
+Borrowed from Windows Search USN Journal change tracking
+(C:\\ProgramData\\Microsoft\\Search\\Data\\Applications\\Windows\\Windows-usn.db):
+Windows Search uses the NTFS USN (Update Sequence Number) journal to detect
+file changes in O(1) instead of scanning the entire filesystem.  The
+ChangeTracking table tracks moves, deletes, and byte-range changes with
+batch-based processing.
+
+Since Python cannot access the raw NTFS USN journal without ctypes bindings,
+this module uses file mtime + size as a watermark — the same principle but
+portable across filesystems.
+
+Pattern: B27 in ALGO.md.
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from enum import IntFlag
+from pathlib import Path
+from typing import Any, Iterable
+
+
+class ChangeType(IntFlag):
+    NONE = 0
+    CREATED = 1
+    MODIFIED = 2
+    DELETED = 4
+    MOVED = 8
+
+
+@dataclass
+class FileWatermark:
+    """Per-file watermark — mtime + size + inode."""
+    path: str
+    mtime: float
+    size: int
+    inode: int = 0  # for move detection on platforms that support it
+
+    def matches(self, stat: os.stat_result) -> bool:
+        return (self.mtime == stat.st_mtime
+                and self.size == stat.st_size)
+
+
+@dataclass
+class ChangeRecord:
+    """A single detected change."""
+    path: str
+    change_type: ChangeType
+    old_path: str | None = None  # for moves
+    watermark: FileWatermark | None = None
+    timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class IncrementalIndex:
+    """Mtime-watermark based incremental change tracker.
+
+    Usage:
+        tracker = IncrementalIndex.load(watermark_file)
+        changes = tracker.scan(directory, glob="*.py")
+        for change in changes:
+            if change.change_type & ChangeType.MODIFIED:
+                re_embed(change.path)
+        tracker.save(watermark_file)
+    """
+
+    watermarks: dict[str, FileWatermark] = field(default_factory=dict)
+    last_scan: float = 0.0
+    scan_count: int = 0
+
+    # --- scan -------------------------------------------------------------
+
+    def scan_paths(self, paths: Iterable[Path]) -> list[ChangeRecord]:
+        """Scan an explicit iterable of files and return changes.
+
+        This is the safer companion to ``scan(directory)`` for callers that
+        already have a filtered source list (for example, harness roots with
+        include/exclude patterns).  It preserves the same watermark semantics
+        while avoiding a broad recursive walk.
+        """
+        changes: list[ChangeRecord] = []
+        current_files: dict[str, FileWatermark] = {}
+        current_inodes: set[int] = set()
+
+        for filepath in paths:
+            if not filepath.is_file():
+                continue
+            path_str = str(filepath)
+            try:
+                stat = filepath.stat()
+            except OSError:
+                continue
+
+            wm = FileWatermark(
+                path=path_str,
+                mtime=stat.st_mtime,
+                size=stat.st_size,
+                inode=stat.st_ino,
+            )
+            current_files[path_str] = wm
+            if stat.st_ino:
+                current_inodes.add(stat.st_ino)
+
+            old_wm = self.watermarks.get(path_str)
+            if old_wm is None:
+                moved_from = self._find_by_inode(stat.st_ino, path_str)
+                if moved_from:
+                    changes.append(ChangeRecord(
+                        path=path_str,
+                        change_type=ChangeType.MOVED,
+                        old_path=moved_from,
+                        watermark=wm,
+                    ))
+                else:
+                    changes.append(ChangeRecord(
+                        path=path_str,
+                        change_type=ChangeType.CREATED,
+                        watermark=wm,
+                    ))
+            elif not old_wm.matches(stat):
+                changes.append(ChangeRecord(
+                    path=path_str,
+                    change_type=ChangeType.MODIFIED,
+                    watermark=wm,
+                ))
+
+        for old_path, old_wm in self.watermarks.items():
+            if old_path not in current_files:
+                if old_wm.inode and old_wm.inode in current_inodes:
+                    continue
+                changes.append(ChangeRecord(
+                    path=old_path,
+                    change_type=ChangeType.DELETED,
+                ))
+
+        self.watermarks = current_files
+        self.last_scan = time.time()
+        self.scan_count += 1
+        return changes
+
+    def scan(
+        self,
+        directory: Path,
+        *,
+        glob: str = "**/*",
+        exclude_dirs: set[str] | None = None,
+    ) -> list[ChangeRecord]:
+        """Scan directory and return changes since last scan.
+
+        Detects: created, modified, deleted, moved.
+        """
+        if exclude_dirs is None:
+            exclude_dirs = {".git", "__pycache__", ".venv", "node_modules",
+                            ".mypy_cache", ".pytest_cache", ".ruff_cache"}
+
+        changes: list[ChangeRecord] = []
+        current_files: dict[str, FileWatermark] = {}
+
+        # Walk the directory
+        for filepath in directory.glob(glob):
+            if not filepath.is_file():
+                continue
+            # Skip excluded directories
+            rel_parts = filepath.relative_to(directory).parts
+            if any(part in exclude_dirs for part in rel_parts):
+                continue
+
+            path_str = str(filepath)
+            try:
+                stat = filepath.stat()
+            except OSError:
+                continue
+
+            wm = FileWatermark(
+                path=path_str,
+                mtime=stat.st_mtime,
+                size=stat.st_size,
+                inode=stat.st_ino,
+            )
+            current_files[path_str] = wm
+
+            old_wm = self.watermarks.get(path_str)
+            if old_wm is None:
+                # Check if this file was moved (same inode, different path)
+                moved_from = self._find_by_inode(stat.st_ino, path_str)
+                if moved_from:
+                    changes.append(ChangeRecord(
+                        path=path_str,
+                        change_type=ChangeType.MOVED,
+                        old_path=moved_from,
+                        watermark=wm,
+                    ))
+                else:
+                    changes.append(ChangeRecord(
+                        path=path_str,
+                        change_type=ChangeType.CREATED,
+                        watermark=wm,
+                    ))
+            elif not old_wm.matches(stat):
+                changes.append(ChangeRecord(
+                    path=path_str,
+                    change_type=ChangeType.MODIFIED,
+                    watermark=wm,
+                ))
+
+        # Detect deletions (files in watermarks but not in current_files)
+        for old_path, old_wm in self.watermarks.items():
+            if old_path not in current_files:
+                # Check if it was moved (inode exists under new path)
+                if not self._inode_exists_elsewhere(old_wm.inode, old_path):
+                    changes.append(ChangeRecord(
+                        path=old_path,
+                        change_type=ChangeType.DELETED,
+                    ))
+
+        # Update watermarks
+        self.watermarks = current_files
+        self.last_scan = time.time()
+        self.scan_count += 1
+
+        return changes
+
+    def _find_by_inode(self, inode: int, exclude_path: str) -> str | None:
+        """Find a watermark with the same inode (for move detection)."""
+        if inode == 0:
+            return None
+        for path, wm in self.watermarks.items():
+            if path != exclude_path and wm.inode == inode:
+                return path
+        return None
+
+    def _inode_exists_elsewhere(self, inode: int, exclude_path: str) -> bool:
+        """Check if inode exists in current scan under a different path."""
+        return self._find_by_inode(inode, exclude_path) is not None
+
+    # --- query ------------------------------------------------------------
+
+    def needs_update(self, path: str) -> bool:
+        """Check if a specific file has changed since last scan."""
+        filepath = Path(path)
+        if not filepath.exists():
+            return path in self.watermarks  # was deleted
+        try:
+            stat = filepath.stat()
+        except OSError:
+            return False
+        wm = self.watermarks.get(path)
+        if wm is None:
+            return True  # new file
+        return not wm.matches(stat)
+
+    def get_changed_files(self, directory: Path, **kwargs: Any) -> list[str]:
+        """Return only the paths of changed files (convenience)."""
+        changes = self.scan(directory, **kwargs)
+        return [c.path for c in changes if c.change_type != ChangeType.NONE]
+
+    # --- serialization ----------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "watermarks": {
+                path: {"mtime": wm.mtime, "size": wm.size, "inode": wm.inode}
+                for path, wm in self.watermarks.items()
+            },
+            "last_scan": self.last_scan,
+            "scan_count": self.scan_count,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> IncrementalIndex:
+        tracker = cls()
+        for path, wm_data in data.get("watermarks", {}).items():
+            tracker.watermarks[path] = FileWatermark(
+                path=path,
+                mtime=wm_data["mtime"],
+                size=wm_data["size"],
+                inode=wm_data.get("inode", 0),
+            )
+        tracker.last_scan = data.get("last_scan", 0.0)
+        tracker.scan_count = data.get("scan_count", 0)
+        return tracker
+
+    def save(self, path: Path) -> None:
+        """Save watermarks to a JSON file."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(
+            json.dumps(self.to_dict(), indent=2),
+            encoding="utf-8",
+        )
+
+    @classmethod
+    def load(cls, path: Path) -> IncrementalIndex:
+        """Load watermarks from a JSON file.
+
+        Returns empty tracker if file doesn't exist (first scan).
+        """
+        if not path.exists():
+            return cls()
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            return cls()
+        return cls.from_dict(data)
+
+    # --- stats ------------------------------------------------------------
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            "tracked_files": len(self.watermarks),
+            "last_scan": self.last_scan,
+            "scan_count": self.scan_count,
+        }

algo_cli/intelligence/index_store.py

@@ -0,0 +1,16 @@
+"""Small persistence helpers for project intelligence indexes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .project_graph import ProjectGraph, build_project_graph, load_project_graph
+
+
+def ensure_project_graph(root: Path | str, *, rebuild: bool = False) -> ProjectGraph:
+    root_path = Path(root).resolve()
+    if not rebuild:
+        graph = load_project_graph(root_path)
+        if graph is not None:
+            return graph
+    return build_project_graph(root_path, persist=True)

algo_cli/intelligence/iteration_plan.py

@@ -0,0 +1,133 @@
+"""B39. Iteration Plan + Endgame Checklist Cadence (VS Code Project Pattern).
+
+Manages monthly iteration plans and release endgame checklists so
+development stays disciplined as features accumulate.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+
+class CheckState(Enum):
+    UNCHECKED = "unchecked"
+    PASSED = "passed"
+    FAILED = "failed"
+    BLOCKED = "blocked"
+
+
+@dataclass
+class ChecklistItem:
+    name: str
+    description: str = ""
+    state: CheckState = CheckState.UNCHECKED
+    blocker: bool = False  # if True, failed state blocks release
+
+
+@dataclass
+class IterationPlan:
+    month: str  # "YYYY-MM"
+    goals: list[str] = field(default_factory=list)
+    work_items: list[dict[str, str]] = field(default_factory=list)  # {label, kind, status}
+    created_at: str = field(default_factory=lambda: datetime.now().isoformat())
+
+
+@dataclass
+class EndgameChecklist:
+    release_version: str
+    items: list[ChecklistItem] = field(default_factory=list)
+    created_at: str = field(default_factory=lambda: datetime.now().isoformat())
+
+    def add_item(self, name: str, description: str = "", blocker: bool = False) -> None:
+        self.items.append(ChecklistItem(name=name, description=description, blocker=blocker))
+
+    def mark(self, name: str, state: CheckState) -> bool:
+        for item in self.items:
+            if item.name == name:
+                item.state = state
+                return True
+        return False
+
+    def release_ready(self) -> bool:
+        """True if no blocker item is failed or unchecked."""
+        for item in self.items:
+            if item.blocker and item.state != CheckState.PASSED:
+                return False
+        return True
+
+    def blockers(self) -> list[ChecklistItem]:
+        """Return unresolved blocker items."""
+        return [i for i in self.items if i.blocker and i.state != CheckState.PASSED]
+
+    def summary(self) -> dict[str, Any]:
+        total = len(self.items)
+        passed = sum(1 for i in self.items if i.state == CheckState.PASSED)
+        failed = sum(1 for i in self.items if i.state == CheckState.FAILED)
+        unchecked = sum(1 for i in self.items if i.state == CheckState.UNCHECKED)
+        blocked = sum(1 for i in self.items if i.state == CheckState.BLOCKED)
+        return {
+            "release": self.release_version,
+            "total": total,
+            "passed": passed,
+            "failed": failed,
+            "unchecked": unchecked,
+            "blocked": blocked,
+            "release_ready": self.release_ready(),
+            "blockers": [b.name for b in self.blockers()],
+        }
+
+
+def default_endgame_checklist(version: str) -> EndgameChecklist:
+    """Create a standard release endgame checklist."""
+    cl = EndgameChecklist(release_version=version)
+    cl.add_item("tests_pass", "All tests pass (pytest -q)", blocker=True)
+    cl.add_item("no_skip_markers", "No new pytest.skip markers without justification", blocker=False)
+    cl.add_item("help_updated", "/help text updated for new commands", blocker=True)
+    cl.add_item("docs_updated", "ALGO.md and README updated", blocker=False)
+    cl.add_item("migration_notes", "Migration notes for breaking changes", blocker=True)
+    cl.add_item("safety_gates", "Safety gates verified (external send, delete, payments)", blocker=True)
+    cl.add_item("rollback_plan", "Rollback plan documented", blocker=False)
+    cl.add_item("changelog", "Changelog/changefiles written", blocker=False)
+    return cl
+
+
+class IterationPlanManager:
+    """Manages iteration plans and endgame checklists."""
+
+    def __init__(self) -> None:
+        self.plans: dict[str, IterationPlan] = {}
+        self.checklists: dict[str, EndgameChecklist] = {}
+
+    def create_plan(self, month: str, goals: list[str] | None = None) -> IterationPlan:
+        plan = IterationPlan(month=month, goals=goals or [])
+        self.plans[month] = plan
+        return plan
+
+    def add_work_item(self, month: str, label: str, kind: str, status: str = "open") -> bool:
+        plan = self.plans.get(month)
+        if not plan:
+            return False
+        plan.work_items.append({"label": label, "kind": kind, "status": status})
+        return True
+
+    def create_checklist(self, version: str) -> EndgameChecklist:
+        cl = default_endgame_checklist(version)
+        self.checklists[version] = cl
+        return cl
+
+    def current_plan(self) -> IterationPlan | None:
+        if not self.plans:
+            return None
+        return self.plans[max(self.plans.keys())]
+
+    def roadmap_summary(self) -> dict[str, Any]:
+        current = self.current_plan()
+        return {
+            "current_month": current.month if current else None,
+            "goals": current.goals if current else [],
+            "open_items": sum(1 for w in (current.work_items if current else []) if w.get("status") == "open"),
+            "checklists": {v: cl.summary() for v, cl in self.checklists.items()},
+        }

algo_cli/intelligence/kernel_plugins.py

@@ -0,0 +1,167 @@
+"""B45. Kernel Plugin Architecture with Typed Annotations (Semantic Kernel Pattern).
+
+Tools are plain Python methods decorated with @kernel_function and typed
+with Annotated[T, "description"].  The kernel introspects the function
+signature and auto-generates the tool schema — no manual JSON schema
+authoring.  Structured outputs use Pydantic BaseModel with response_format.
+"""
+
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass, field
+from typing import Any, Callable, get_type_hints
+
+
+@dataclass
+class ToolParam:
+    name: str
+    type: str
+    description: str
+    required: bool
+
+
+@dataclass
+class ToolSchema:
+    name: str
+    description: str
+    params: list[ToolParam] = field(default_factory=list)
+    return_type: str = "string"
+    return_description: str = ""
+
+
+def kernel_function(description: str = "") -> Callable:
+    """Decorator that marks a method as a kernel-callable tool."""
+    def decorator(fn: Callable) -> Callable:
+        fn._kernel_description = description or fn.__doc__ or ""
+        fn._kernel_function = True
+        return fn
+    return decorator
+
+
+def introspect_tool(fn: Callable) -> ToolSchema:
+    """Auto-generate tool schema from function signature + Annotated hints."""
+    sig = inspect.signature(fn)
+    try:
+        hints = get_type_hints(fn, include_extras=True)
+    except Exception:
+        hints = {}
+
+    params: list[ToolParam] = []
+    for name, param in sig.parameters.items():
+        if name == "self":
+            continue
+        hint = hints.get(name, str)
+        desc = ""
+        type_name = "string"
+
+        # Extract Annotated[T, "description"]
+        if hasattr(hint, "__metadata__"):
+            origin = getattr(hint, "__origin__", str)
+            type_name = getattr(origin, "__name__", "string")
+            if hint.__metadata__:
+                desc = str(hint.__metadata__[0])
+        else:
+            type_name = getattr(hint, "__name__", "string")
+
+        params.append(ToolParam(
+            name=name,
+            type=type_name,
+            description=desc,
+            required=param.default is inspect.Parameter.empty,
+        ))
+
+    # Return type
+    ret_hint = hints.get("return", type(None))
+    ret_desc = ""
+    if hasattr(ret_hint, "__metadata__"):
+        ret_desc = str(ret_hint.__metadata__[0]) if ret_hint.__metadata__ else ""
+    ret_origin = getattr(ret_hint, "__origin__", ret_hint)
+    ret_type = getattr(ret_origin, "__name__", "void")
+
+    return ToolSchema(
+        name=fn.__name__,
+        description=getattr(fn, "_kernel_description", fn.__doc__ or ""),
+        params=params,
+        return_type=ret_type,
+        return_description=ret_desc,
+    )
+
+
+class Kernel:
+    """Central kernel that manages plugins and invokes tools."""
+
+    def __init__(self) -> None:
+        self._plugins: dict[str, Any] = {}
+        self._schemas: dict[str, ToolSchema] = {}
+        self._handlers: dict[str, Callable] = {}
+
+    def add_plugin(self, plugin: Any, name: str | None = None) -> None:
+        """Register a plugin class — all @kernel_function methods become tools."""
+        plugin_name = name or plugin.__class__.__name__
+        self._plugins[plugin_name] = plugin
+        for attr_name in dir(plugin):
+            attr = getattr(plugin, attr_name, None)
+            if callable(attr) and getattr(attr, "_kernel_function", False):
+                schema = introspect_tool(attr)
+                self._schemas[schema.name] = schema
+                self._handlers[schema.name] = attr
+
+    def list_tools(self) -> list[ToolSchema]:
+        return list(self._schemas.values())
+
+    def get_tool_schema(self, name: str) -> ToolSchema | None:
+        return self._schemas.get(name)
+
+    def invoke(self, tool_name: str, **kwargs: Any) -> str:
+        """Invoke a registered tool by name."""
+        handler = self._handlers.get(tool_name)
+        if not handler:
+            return f"Tool '{tool_name}' not found"
+        try:
+            result = handler(**kwargs)
+            return str(result)
+        except Exception as e:
+            return f"Tool '{tool_name}' error: {e}"
+
+    def tool_descriptions_for_prompt(self) -> str:
+        """Generate tool description text for inclusion in system prompt."""
+        lines: list[str] = []
+        for schema in self._schemas.values():
+            param_str = ", ".join(
+                f"{p.name}: {p.type}" + (f" — {p.description}" if p.description else "")
+                for p in schema.params
+            )
+            lines.append(f"- {schema.name}({param_str}): {schema.description}")
+        return "\n".join(lines)
+
+
+# ── structured output support ─────────────────────────────────────────
+
+
+def validate_structured_output(data: dict, model_class: type) -> dict:
+    """Validate LLM output against a Pydantic-like BaseModel.
+
+    Works with any class that has __annotations__ (dataclass or Pydantic).
+    """
+    try:
+        from pydantic import BaseModel
+        if issubclass(model_class, BaseModel):
+            obj = model_class(**data)
+            return {"valid": True, "data": obj.model_dump()}
+    except ImportError:
+        pass
+    except Exception as e:
+        return {"valid": False, "error": str(e)}
+
+    # Fallback: simple annotation-based validation
+    annotations = getattr(model_class, "__annotations__", {})
+    errors: list[str] = []
+    for field_name, field_type in annotations.items():
+        if field_name not in data:
+            errors.append(f"missing field: {field_name}")
+        elif not isinstance(data[field_name], field_type):
+            errors.append(f"{field_name}: expected {field_type.__name__}, got {type(data[field_name]).__name__}")
+    if errors:
+        return {"valid": False, "error": "; ".join(errors)}
+    return {"valid": True, "data": data}