source-kb 0.2.2__py3-none-any.whl

core/git.py

@@ -0,0 +1,294 @@
+"""Git operations wrapper — clone, fetch, diff, file reading.
+
+Provides retry, timeout, and input validation for all git commands.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import time
+from pathlib import Path
+
+
+# ─── Errors ─────────────────────────────────────────────────
+
+class GitError(Exception):
+    pass
+
+class GitAuthError(GitError):
+    pass
+
+class GitNetworkError(GitError):
+    pass
+
+class GitRepoNotFoundError(GitError):
+    pass
+
+class GitBranchNotFoundError(GitError):
+    pass
+
+class ConfigError(GitError):
+    pass
+
+
+# ─── Validation ─────────────────────────────────────────────
+
+_SAFE_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]*$")
+_SAFE_URL = re.compile(r"^(?:https?://|git://|ssh://|git@)[^\s]+$")
+_SAFE_REF = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._/~^{}\-]*$")
+
+
+def _validate_name(name: str, label: str = "name"):
+    if not name or not _SAFE_NAME.match(name) or ".." in name:
+        raise ConfigError(f"Invalid {label}: {name!r}")
+
+
+def _validate_url(url: str):
+    if not url:
+        return
+    if url.startswith("-"):
+        raise ConfigError(f"Invalid URL: {url!r}")
+    if not _SAFE_URL.match(url):
+        raise ConfigError(f"Invalid URL: {url!r}")
+
+
+def _validate_ref(ref: str):
+    if not ref or ref.startswith("-"):
+        raise ConfigError(f"Invalid ref: {ref!r}")
+    if "/.." in ref or "../" in ref:
+        raise ConfigError(f"Invalid ref: {ref!r}")
+    if not _SAFE_REF.match(ref):
+        raise ConfigError(f"Invalid ref: {ref!r}")
+
+
+# ─── Core ───────────────────────────────────────────────────
+
+def git(repo_path: str | Path, *args, timeout: int = 60) -> tuple[int, str, str]:
+    """Execute a git command. Returns (returncode, stdout, stderr)."""
+    cmd = ["git", "-C", str(repo_path)] + list(args)
+    try:
+        r = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", timeout=timeout)
+        return r.returncode, r.stdout, r.stderr
+    except subprocess.TimeoutExpired:
+        raise GitNetworkError(f"Git command timed out: {' '.join(cmd)}")
+    except FileNotFoundError:
+        raise GitError("git not found — please install git")
+
+
+# ─── Repository management ──────────────────────────────────
+
+def fetch_with_retry(repo_path: Path, branch: str = "main", max_retries: int = 3) -> bool:
+    """Git fetch with exponential backoff retry."""
+    _validate_ref(branch)
+    for attempt in range(max_retries):
+        code, _, stderr = git(repo_path, "fetch", "origin", branch)
+        if code == 0:
+            return True
+        if code == 128 and ("authentication" in stderr.lower() or "permission denied" in stderr.lower()):
+            raise GitAuthError(f"Authentication failed: {stderr.strip()}")
+        if attempt < max_retries - 1:
+            time.sleep(2 ** (attempt + 1))
+        else:
+            raise GitNetworkError(f"Fetch failed after {max_retries} retries: {stderr.strip()}")
+    return False
+
+
+def clone_with_retry(url: str, target: Path, branch: str = "main", max_retries: int = 3) -> bool:
+    """Git clone with retry. Returns True on success."""
+    _validate_url(url)
+    if (target / ".git").exists():
+        return True
+    target.parent.mkdir(parents=True, exist_ok=True)
+
+    for attempt in range(max_retries):
+        cmd = ["git", "clone", "--depth", "1", "--single-branch", "-b", branch, "--", url, str(target)]
+        try:
+            r = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", timeout=300)
+        except subprocess.TimeoutExpired:
+            if attempt >= max_retries - 1:
+                raise GitNetworkError(f"Clone timed out after {max_retries} retries")
+            time.sleep(2 ** (attempt + 1))
+            continue
+
+        if r.returncode == 0:
+            return True
+        err = r.stderr.lower()
+        if r.returncode == 128:
+            if "authentication" in err or "permission denied" in err:
+                raise GitAuthError(f"Authentication failed: {r.stderr.strip()}")
+            if "not found" in err or "does not exist" in err:
+                raise GitRepoNotFoundError(f"Repository not found: {url}")
+            if "branch" in err:
+                raise GitBranchNotFoundError(f"Branch not found: {branch}")
+        # Clean partial clone
+        if target.exists():
+            import shutil
+            shutil.rmtree(target, ignore_errors=True)
+        if attempt < max_retries - 1:
+            time.sleep(2 ** (attempt + 1))
+        else:
+            raise GitNetworkError(f"Clone failed after {max_retries} retries: {r.stderr.strip()}")
+    return False
+
+
+def ensure_repo(url: str | None, local: str | None, cache_dir: Path, module_name: str, branch: str = "main") -> Path:
+    """Ensure a local repo is available (clone or use existing)."""
+    _validate_name(module_name, "module_name")
+    target = cache_dir / module_name
+
+    if (target / ".git").exists():
+        fetch_with_retry(target, branch)
+        return target
+    if url:
+        _validate_url(url)
+        clone_with_retry(url, target, branch)
+        return target
+    if local:
+        local_path = Path(local).expanduser()
+        if not (local_path / ".git").exists():
+            raise ConfigError(f"Not a git repo: {local_path}")
+        return local_path
+    raise ConfigError(f"Module {module_name} has neither url nor local configured")
+
+
+# ─── File reading ───────────────────────────────────────────
+
+def read_file(repo_path: Path, ref: str, filepath: str) -> str | None:
+    """Read file content at a given ref. Returns None if not found."""
+    _validate_ref(ref)
+    code, stdout, stderr = git(repo_path, "show", f"{ref}:{filepath}")
+    if code == 0:
+        return stdout
+    if "does not exist" in stderr or "not found" in stderr:
+        return None
+    raise GitError(f"Failed to read file: {stderr.strip()}")
+
+
+def ls_tree(repo_path: Path, ref: str = "origin/main") -> list[str]:
+    """List all files at a given ref."""
+    _validate_ref(ref)
+    code, stdout, stderr = git(repo_path, "ls-tree", "-r", "--name-only", ref)
+    if code != 0:
+        raise GitError(f"ls-tree failed: {stderr.strip()}")
+    return [l.strip() for l in stdout.splitlines() if l.strip()]
+
+
+# ─── Diff operations ────────────────────────────────────────
+
+def get_head_commit(repo_path: Path, ref: str = "origin/main") -> str | None:
+    """Get commit hash for a ref. Returns None if unknown."""
+    _validate_ref(ref)
+    code, stdout, _ = git(repo_path, "rev-parse", ref)
+    return stdout.strip() if code == 0 else None
+
+
+def diff_files(repo_path: Path, old_commit: str, new_commit: str) -> list[dict]:
+    """Get changed files between two commits. Returns [{file, status, old_path?, new_path?}]."""
+    _validate_ref(old_commit)
+    _validate_ref(new_commit)
+    code, stdout, stderr = git(repo_path, "diff", "--name-status", f"{old_commit}..{new_commit}")
+    if code != 0:
+        raise GitError(f"diff failed: {stderr.strip()}")
+
+    changes = []
+    for line in stdout.splitlines():
+        if not line.strip():
+            continue
+        parts = line.split("\t")
+        status = parts[0][0]
+        if status == "R" and len(parts) >= 3:
+            changes.append({"status": "R", "old_path": parts[1], "new_path": parts[2], "file": parts[2]})
+        elif len(parts) >= 2:
+            changes.append({"status": status, "file": parts[1]})
+    return changes
+
+
+def diff_stat(repo_path: Path, old_commit: str, new_commit: str) -> str:
+    """Get shortstat between two commits."""
+    _validate_ref(old_commit)
+    _validate_ref(new_commit)
+    code, stdout, stderr = git(repo_path, "diff", "--shortstat", f"{old_commit}..{new_commit}")
+    if code != 0:
+        raise GitError(f"diff stat failed: {stderr.strip()}")
+    return stdout.strip()
+
+
+def detect_changes(repo_path: Path, from_ref: str, to_ref: str) -> list[dict]:
+    """Detect changes between two refs with human-readable status.
+
+    Returns list of {"path": str, "status": "added"|"modified"|"deleted"|"renamed"}.
+    """
+    _STATUS_MAP = {"A": "added", "M": "modified", "D": "deleted", "R": "renamed", "C": "copied", "T": "modified"}
+    raw = diff_files(repo_path, from_ref, to_ref)
+    result = []
+    for entry in raw:
+        status = _STATUS_MAP.get(entry["status"], "modified")
+        result.append({"path": entry["file"], "status": status})
+    return result
+
+
+def get_commit_log(
+    repo_path: Path, from_ref: str, to_ref: str, max_count: int = 50
+) -> list[dict]:
+    """Get commit log between two refs.
+
+    Returns list of {"hash": str, "author": str, "date": str, "message": str}.
+    """
+    _validate_ref(from_ref)
+    _validate_ref(to_ref)
+    fmt = "%H%n%an%n%aI%n%s%n---"
+    code, stdout, stderr = git(
+        repo_path, "log", f"--max-count={max_count}", f"--format={fmt}", f"{from_ref}..{to_ref}"
+    )
+    if code != 0:
+        raise GitError(f"git log failed: {stderr.strip()}")
+    commits = []
+    lines = stdout.strip().split("\n")
+    i = 0
+    while i + 3 < len(lines):
+        commits.append({
+            "hash": lines[i].strip(),
+            "author": lines[i + 1].strip(),
+            "date": lines[i + 2].strip(),
+            "message": lines[i + 3].strip(),
+        })
+        # skip separator line "---"
+        i += 5
+    return commits
+
+
+def clone_repo(
+    url: str, dest: Path, branch: str = "main", auth: dict | None = None
+) -> Path:
+    """Clone a repository with optional authentication.
+
+    Args:
+        url: Repository URL.
+        dest: Destination path.
+        branch: Branch to clone.
+        auth: Optional dict with "type" ("ssh_key"|"token") and credentials.
+              - {"type": "ssh_key", "key_path": "/path/to/key"}
+              - {"type": "token", "token": "ghp_xxx"}
+
+    Returns:
+        Path to the cloned repository.
+    """
+    _validate_url(url)
+    if auth:
+        auth_type = auth.get("type", "")
+        if auth_type == "token" and auth.get("token"):
+            # Inject token into HTTPS URL
+            token = auth["token"]
+            if url.startswith("https://"):
+                url = url.replace("https://", f"https://x-access-token:{token}@", 1)
+        elif auth_type == "ssh_key" and auth.get("key_path"):
+            key_path = auth["key_path"]
+            import os
+            os.environ["GIT_SSH_COMMAND"] = f'ssh -i {key_path} -o StrictHostKeyChecking=no'
+    clone_with_retry(url, dest, branch)
+    # Clean up SSH env if set
+    if auth and auth.get("type") == "ssh_key":
+        import os
+        os.environ.pop("GIT_SSH_COMMAND", None)
+    return dest

core/interfaces.py

@@ -0,0 +1,249 @@
+"""Abstract interfaces and shared data classes for the core package.
+
+All ABCs and dataclasses that define contracts between layers live here.
+No imports from cli/ or skill/ are permitted.
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# LLM data transfer objects
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LlmRequest:
+    """Encapsulates a single LLM call request."""
+
+    system: str
+    user: str
+    model: str | None = None  # None = use default from config
+    max_tokens: int = 8192
+    temperature: float = 0.3
+
+
+@dataclass
+class LlmResponse:
+    """Encapsulates the result of an LLM call."""
+
+    content: str
+    status: Literal["done", "delegated", "dry-run", "failed"]
+    usage: dict[str, int] = field(default_factory=dict)
+    elapsed: float = 0.0
+    error: str = ""
+    prompt_file: str = ""  # For delegated mode
+
+
+# ---------------------------------------------------------------------------
+# LlmStrategy ABC
+# ---------------------------------------------------------------------------
+
+
+class LlmStrategy(ABC):
+    """Abstract LLM execution backend.
+
+    Implementations: ApiLlmStrategy, DelegatedLlmStrategy, DryRunLlmStrategy.
+    """
+
+    @abstractmethod
+    def call(self, request: LlmRequest) -> LlmResponse: ...
+
+    def call_batch(
+        self, requests: list[LlmRequest], max_concurrent: int = 5
+    ) -> list[LlmResponse]:
+        """Execute multiple requests. Default: sequential. CLI overrides with ThreadPoolExecutor."""
+        return [self.call(r) for r in requests]
+
+
+# ---------------------------------------------------------------------------
+# PromptAssembler ABC
+# ---------------------------------------------------------------------------
+
+
+class PromptAssembler(ABC):
+    """Abstract prompt content assembly strategy.
+
+    CLI injects InlineAssembler (inlines source); Agent uses ReferenceAssembler
+    (emits file path references).
+    """
+
+    @abstractmethod
+    def resolve_file_list(self, module_dir: Path, doc_type: str,
+                         file_list_override: str | None = None) -> str: ...
+
+    @abstractmethod
+    def resolve_source_content(
+        self, module_dir: Path, doc_type: str, source_cache: Path
+    ) -> str: ...
+
+    def resolve_source_content_from_paths(
+        self, module_dir: Path, doc_type: str, source_cache: Path, file_paths: list[str]
+    ) -> str:
+        """Resolve source content from an explicit file path list (shard override).
+
+        Default implementation falls back to resolve_source_content.
+        Subclasses may override for optimized shard handling.
+        """
+        return self.resolve_source_content(module_dir, doc_type, source_cache)
+
+    @abstractmethod
+    def resolve_skeleton_content(self, module_dir: Path) -> str: ...
+
+    @abstractmethod
+    def should_append_source(self) -> bool: ...
+
+
+# ---------------------------------------------------------------------------
+# SkeletonParser ABC
+# ---------------------------------------------------------------------------
+
+
+class SkeletonParser(ABC):
+    """Language-specific skeleton parser. Registered by name in parser registry."""
+
+    name: str
+    priority: int  # Higher = preferred (jqassistant=100, treesitter=80, regex=50)
+
+    @abstractmethod
+    def can_parse(self, repo_path: Path, preset: dict[str, Any]) -> bool: ...
+
+    @abstractmethod
+    def parse(
+        self, repo_path: Path, preset: dict[str, Any], **kwargs: Any
+    ) -> list[dict[str, Any]]: ...
+
+
+# ---------------------------------------------------------------------------
+# Step ABC and StepResult
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class StepResult:
+    """Outcome of a single pipeline step execution."""
+
+    status: Literal["ok", "failed", "skipped", "delegated"]
+    message: str = ""
+    details: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return self.status == "ok"
+
+    @property
+    def skipped(self) -> bool:
+        return self.status == "skipped"
+
+    @property
+    def delegated(self) -> bool:
+        return self.status == "delegated"
+
+    def __str__(self) -> str:
+        return f"[{self.status.upper()}] {self.message}"
+
+
+class Step(ABC):
+    """Abstract pipeline step with optional checkpoint and rollback."""
+
+    name: str
+    checkpoint: str | None = None
+
+    def __init__(self, name: str, checkpoint: str | None = None):
+        self.name = name
+        self.checkpoint = checkpoint
+
+    @abstractmethod
+    def run(self, ctx: PipelineContext) -> StepResult: ...
+
+    def rollback(self, ctx: PipelineContext) -> None:
+        """Optional cleanup on failure. Default is no-op."""
+
+    def __repr__(self) -> str:
+        cp = f" (cp={self.checkpoint})" if self.checkpoint else ""
+        return f"<{self.__class__.__name__} '{self.name}'{cp}>"
+
+
+# ---------------------------------------------------------------------------
+# PipelineContext
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PipelineContext:
+    """Shared state carried across pipeline steps."""
+
+    config: dict[str, Any]
+    kb_name: str
+    kb_config: dict[str, Any]
+    knowledge_dir: Path
+    project_root: Path
+    module: str | None = None
+    cache_dir: Path = field(default_factory=lambda: Path(".source-cache"))
+    state: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_config(
+        cls,
+        config: dict[str, Any],
+        kb_name: str,
+        *,
+        module: str | None = None,
+    ) -> PipelineContext:
+        """Construct context from a loaded configuration dict."""
+        kb_config = config["knowledge_bases"][kb_name]
+        knowledge_dir = Path(kb_config["knowledge_dir"])
+        source = kb_config.get("source", {})
+        cache_dir = Path(source.get("cache_dir", "./.source-cache"))
+        project_root = Path.cwd()
+
+        return cls(
+            config=config,
+            kb_name=kb_name,
+            kb_config=kb_config,
+            knowledge_dir=knowledge_dir,
+            project_root=project_root,
+            module=module,
+            cache_dir=cache_dir,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Validator ABC and ValidationResult
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ValidationResult:
+    """Structured output from a validator run."""
+
+    errors: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return len(self.errors) == 0
+
+    def merge(self, other: ValidationResult) -> ValidationResult:
+        """Combine two results into one."""
+        return ValidationResult(
+            errors=self.errors + other.errors,
+            warnings=self.warnings + other.warnings,
+        )
+
+
+class Validator(ABC):
+    """Abstract document quality validator. Registered by name."""
+
+    name: str
+
+    @abstractmethod
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult: ...

core/monitor/__init__.py

@@ -0,0 +1,5 @@
+"""core.monitor — progress monitoring and heartbeat detection.
+
+Provides structured progress file read/write and heartbeat timeout detection.
+Used by both CLI (automatic heartbeat) and Agent (manual progress check).
+"""

core/monitor/progress.py

@@ -0,0 +1,83 @@
+"""Progress monitoring — read/write progress files, detect heartbeat timeouts.
+
+Usage:
+    from core.monitor.progress import write_progress, read_progress, check_heartbeat
+
+    write_progress(module_dir, "business-logic", "RUNNING")
+    status = read_progress(module_dir, "business-logic")
+    stale = check_heartbeat(module_dir, timeout_seconds=60)
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+from core.paths import progress_path, progress_dir, ensure_dir
+
+
+def write_progress(module_dir: Path, doc_name: str, message: str) -> None:
+    """Write a progress status message."""
+    path = progress_path(module_dir, doc_name)
+    ensure_dir(path.parent)
+    ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    path.write_text(f"{message} ({ts})", encoding="utf-8")
+
+
+def read_progress(module_dir: Path, doc_name: str) -> str | None:
+    """Read current progress status. Returns None if no progress file."""
+    path = progress_path(module_dir, doc_name)
+    if path.exists():
+        return path.read_text(encoding="utf-8").strip()
+    return None
+
+
+def check_heartbeat(module_dir: Path, timeout_seconds: int = 60) -> list[str]:
+    """Find progress files with stale heartbeats (no update within timeout).
+
+    Returns list of doc_names that appear stale/stuck.
+    """
+    stale: list[str] = []
+    prog_dir = progress_dir(module_dir)
+    if not prog_dir.is_dir():
+        return stale
+
+    now = time.time()
+    for f in prog_dir.iterdir():
+        if not f.is_file() or f.name.endswith(".hb-pid"):
+            continue
+        mtime = f.stat().st_mtime
+        if now - mtime > timeout_seconds:
+            content = f.read_text(encoding="utf-8").strip()
+            # Only flag if still in RUNNING state
+            if "RUNNING" in content or "HEARTBEAT" in content:
+                stale.append(f.name)
+
+    return stale
+
+
+def cleanup_progress(module_dir: Path) -> int:
+    """Remove all progress files (after successful completion). Returns count removed."""
+    prog_dir = progress_dir(module_dir)
+    if not prog_dir.is_dir():
+        return 0
+    count = 0
+    for f in prog_dir.iterdir():
+        f.unlink(missing_ok=True)
+        count += 1
+    return count
+
+
+def check_progress(module_dir: Path) -> dict[str, str]:
+    """Read all progress files and return {doc_name: status} mapping.
+
+    Returns empty dict if no progress directory or no progress files.
+    """
+    prog_dir = progress_dir(module_dir)
+    if not prog_dir.is_dir():
+        return {}
+    result: dict[str, str] = {}
+    for f in prog_dir.iterdir():
+        if f.is_file() and not f.name.endswith(".hb-pid"):
+            result[f.stem] = f.read_text(encoding="utf-8").strip()
+    return result

core/monitor/prompt_store.py

@@ -0,0 +1,49 @@
+"""Prompt persistence — save rendered prompts to disk for debugging.
+
+Usage:
+    from core.monitor.prompt_store import save_prompt
+
+    save_prompt(module_dir, task_id, system_prompt, user_prompt)
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+
+def save_prompt(
+    module_dir: Path,
+    task_id: str,
+    system: str,
+    user: str,
+    *,
+    model: str = "",
+) -> Path:
+    """Persist a rendered prompt to .meta/prompts/{task_id}.md.
+
+    Returns the path to the saved file.
+    """
+    prompts_dir = module_dir / ".meta" / "prompts"
+    prompts_dir.mkdir(parents=True, exist_ok=True)
+
+    safe_id = task_id.replace("/", "_").replace("\\", "_")
+    path = prompts_dir / f"{safe_id}.md"
+
+    ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    header = f"<!-- task_id: {task_id} | model: {model} | saved: {ts} -->\n\n"
+    content = header
+    if system:
+        content += f"## System Prompt\n\n{system}\n\n"
+    content += f"## User Prompt\n\n{user}\n"
+
+    path.write_text(content, encoding="utf-8")
+    return path
+
+
+def should_save_prompts(config: dict) -> bool:
+    """Check if prompt persistence is enabled in config."""
+    debug = config.get("debug", {})
+    if isinstance(debug, dict):
+        return bool(debug.get("save_prompts", False))
+    return False