source-kb 0.2.2__py3-none-any.whl

cli/commands/validate.py

@@ -0,0 +1,191 @@
+"""source-kb validate — Document validation (coverage, consistency, links, structure, sampling)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def register(subparsers: argparse._SubParsersAction) -> None:
+    p = subparsers.add_parser("validate", help="Run validation checks on knowledge base docs")
+    vsub = p.add_subparsers(dest="validator")
+
+    # coverage
+    c = vsub.add_parser("coverage", help="Coverage check")
+    c.add_argument("action", nargs="?", default="check", choices=["check"])
+    c.add_argument("--skeleton", help="Skeleton JSON path")
+    c.add_argument("--skeleton-dir", help="Skeleton shards directory")
+    c.add_argument("--module-dir", help="Module directory")
+    c.add_argument("--docs-dir", help="Documents directory")
+    c.add_argument("--type", default="service", help="Module type")
+    c.set_defaults(func=run_coverage)
+
+    # consistency
+    c = vsub.add_parser("consistency", help="Consistency and progress check")
+    c.add_argument("--module-dir", required=True, help="Module directory")
+    c.add_argument("--preset", help="Preset name")
+    c.add_argument("--source-cache", help="Source cache path")
+    c.add_argument("--cleanup", action="store_true", help="Clean up progress files")
+    c.set_defaults(func=run_consistency)
+
+    # links
+    c = vsub.add_parser("links", help="Link and reference health check")
+    c.add_argument("--module-dir", required=True, help="Module directory")
+    c.add_argument("--config", help="kb-project.yaml path")
+    c.set_defaults(func=run_links)
+
+    # structure
+    c = vsub.add_parser("structure", help="Document structure validation")
+    c.add_argument("--module-dir", required=True, help="Module directory")
+    c.add_argument("--skeleton-dir", help="Skeleton directory for comparison")
+    c.set_defaults(func=run_structure)
+
+    # sampling
+    c = vsub.add_parser("sampling", help="Numeric sampling validation")
+    c.add_argument("--module-dir", required=True, help="Module directory")
+    c.set_defaults(func=run_sampling)
+
+    p.set_defaults(func=_no_validator)
+
+
+def _no_validator(args: argparse.Namespace) -> None:
+    print("Error: specify a validator (coverage, consistency, links, structure, sampling)",
+          file=sys.stderr)
+    sys.exit(1)
+
+
+def run_coverage(args: argparse.Namespace) -> None:
+    from core.validators.coverage import CoverageValidator
+
+    module_dir = Path(args.module_dir) if args.module_dir else None
+    skeleton_path = Path(args.skeleton) if args.skeleton else None
+    skeleton_dir = Path(args.skeleton_dir) if args.skeleton_dir else None
+    docs_dir = Path(args.docs_dir) if args.docs_dir else module_dir
+
+    if not docs_dir:
+        print("Error: --docs-dir or --module-dir required", file=sys.stderr)
+        sys.exit(1)
+
+    validator = CoverageValidator()
+    result = validator.validate(
+        docs_dir,
+        skeleton_path=str(skeleton_path) if skeleton_path else None,
+        skeleton_dir=str(skeleton_dir) if skeleton_dir else None,
+        module_type=args.type or "service",
+    )
+
+    if result.errors:
+        print("ERRORS:")
+        for e in result.errors:
+            print(f"  {e}")
+    if result.warnings:
+        print("WARNINGS:")
+        for w in result.warnings:
+            print(f"  {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings)}, ensure_ascii=False), file=sys.stderr)
+
+
+def run_consistency(args: argparse.Namespace) -> None:
+    from core.validators.consistency import ConsistencyValidator
+    from core.monitor.progress import check_progress, cleanup_progress
+
+    module_dir = Path(args.module_dir)
+
+    if args.cleanup:
+        cleaned = cleanup_progress(module_dir)
+        print(f"Cleaned {cleaned} progress files")
+        print(json.dumps({"status": "ok", "cleaned": cleaned}, ensure_ascii=False), file=sys.stderr)
+        return
+
+    progress = check_progress(module_dir)
+    if progress:
+        print("Progress status:")
+        for doc_name, doc_status in progress.items():
+            print(f"  {doc_name}: {doc_status}")
+
+    validator = ConsistencyValidator()
+    source_cache = Path(args.source_cache) if args.source_cache else None
+    result = validator.validate(module_dir, source_cache=source_cache)
+
+    if result.warnings:
+        print("\nConsistency warnings:")
+        for w in result.warnings:
+            print(f"  {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings),
+                      "progress": progress or {}}, ensure_ascii=False), file=sys.stderr)
+
+
+def run_links(args: argparse.Namespace) -> None:
+    from core.validators.links import LinksValidator
+
+    module_dir = Path(args.module_dir)
+    validator = LinksValidator()
+    result = validator.validate(module_dir)
+
+    if result.errors:
+        print("Broken links:")
+        for e in result.errors:
+            print(f"  {e}")
+    if result.warnings:
+        for w in result.warnings:
+            print(f"  {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings)}, ensure_ascii=False), file=sys.stderr)
+
+
+def run_structure(args: argparse.Namespace) -> None:
+    from core.validators.structure import StructureValidator
+
+    module_dir = Path(args.module_dir)
+    skeleton_dir = Path(args.skeleton_dir) if args.skeleton_dir else None
+
+    validator = StructureValidator()
+    result = validator.validate(module_dir, skeleton_dir=skeleton_dir)
+
+    if result.errors:
+        print("Structure errors:")
+        for e in result.errors:
+            print(f"  {e}")
+    if result.warnings:
+        for w in result.warnings:
+            print(f"  {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings)}, ensure_ascii=False), file=sys.stderr)
+
+
+def run_sampling(args: argparse.Namespace) -> None:
+    from core.validators.sampling import SamplingValidator
+
+    module_dir = Path(args.module_dir)
+    validator = SamplingValidator()
+    result = validator.validate(module_dir)
+
+    if result.warnings:
+        print("Sampling warnings:")
+        for w in result.warnings:
+            print(f"  {w}")
+
+    hit_count = 10 - len(result.warnings)
+    total = 10
+    hit_rate = round(hit_count / total * 100, 1) if total > 0 else 100.0
+
+    if not result.warnings:
+        print(f"Sampling passed (hit rate {hit_rate}%)")
+    else:
+        print(f"Sampling hit rate {hit_rate}% ({len(result.warnings)} misses)")
+
+    status = "ok" if result.passed else "warn"
+    print(json.dumps({"status": status, "warnings": len(result.warnings),
+                      "hit_rate_pct": hit_rate}, ensure_ascii=False), file=sys.stderr)

core/__init__.py

@@ -0,0 +1,32 @@
+"""core — shared algorithm library for source-kb.
+
+This package provides pure functions and tools for knowledge base generation.
+It has no opinion on pipeline orchestration, no global state, and no dependency
+on cli/ or skill/ packages.
+
+Layers:
+    core.config      — Configuration loading and validation
+    core.git         — Git operations (clone, fetch, diff)
+    core.paths       — Canonical path resolution
+    core.preset      — Preset loading and file classification
+    core.skeleton    — Skeleton extraction, file lists, split planning, merge
+    core.prompt      — Prompt rendering and content assembly
+    core.validators  — Document quality validators
+    core.rag         — Vector index (chunking, embedding, retrieval)
+    core.monitor     — Progress monitoring and heartbeat detection
+    core.docs        — Shared document generation
+"""
+
+import logging
+
+__version__ = "0.2.2"
+
+
+def setup_logging(level: int = logging.INFO) -> None:
+    """Configure logging for the core package."""
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter("[%(name)s] %(message)s"))
+    root = logging.getLogger("core")
+    root.setLevel(level)
+    if not root.handlers:
+        root.addHandler(handler)

core/config.py

@@ -0,0 +1,261 @@
+"""Configuration loading, validation, and typed access.
+
+Merges the responsibilities of engine/config_loader.py and engine/runtime_config.py
+into a single module with no global state. The caller loads config once and passes
+the resulting Config object explicitly.
+
+Usage:
+    from core.config import load_config, find_config
+    config = load_config(find_config())
+    kb = config.get_kb("my-kb")
+"""
+
+from __future__ import annotations
+
+import copy
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+# ---------------------------------------------------------------------------
+# Config dataclass
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Config:
+    """Typed, immutable-ish configuration loaded from kb-project.yaml."""
+
+    raw: dict[str, Any]
+    config_path: Path
+
+    # --- Knowledge bases ---
+
+    @property
+    def kb_names(self) -> list[str]:
+        return list(self.raw.get("knowledge_bases", {}).keys())
+
+    def get_kb(self, name: str) -> dict[str, Any]:
+        kbs = self.raw.get("knowledge_bases", {})
+        if name not in kbs:
+            available = ", ".join(kbs.keys()) if kbs else "none"
+            raise KeyError(f"Knowledge base '{name}' not found. Available: {available}")
+        return kbs[name]
+
+    # --- Embedding ---
+
+    @property
+    def embed_backend(self) -> str:
+        return self.raw.get("embedding", {}).get("backend", "ollama")
+
+    @property
+    def embed_model(self) -> str:
+        return self.raw.get("embedding", {}).get("model", "bge-m3")
+
+    @property
+    def embed_base_url(self) -> str:
+        return self.raw.get("embedding", {}).get("base_url", "http://localhost:11434")
+
+    @property
+    def embed_api_key(self) -> str:
+        return os.getenv("EMBEDDING_API_KEY", "") or self.raw.get("embedding", {}).get("api_key", "")
+
+    # --- Chunking ---
+
+    @property
+    def chunk_size(self) -> int:
+        return self.raw.get("chunking", {}).get("size", 512)
+
+    @property
+    def chunk_overlap(self) -> int:
+        return self.raw.get("chunking", {}).get("overlap", 100)
+
+    @property
+    def chunk_min_length(self) -> int:
+        return self.raw.get("chunking", {}).get("min_length", 50)
+
+    # --- Retrieval ---
+
+    @property
+    def top_k(self) -> int:
+        return self.raw.get("retrieval", {}).get("top_k", 10)
+
+    @property
+    def similarity_threshold(self) -> float:
+        return self.raw.get("retrieval", {}).get("similarity_threshold", 0.6)
+
+    # --- Agent / LLM ---
+
+    @property
+    def agent_model(self) -> str:
+        return os.getenv("LLM_MODEL", "") or self.raw.get("agent", {}).get("model", "")
+
+    @property
+    def agent_api_key(self) -> str:
+        return os.getenv("LLM_API_KEY", "") or self.raw.get("agent", {}).get("api_key", "")
+
+    @property
+    def agent_base_url(self) -> str:
+        return os.getenv("LLM_BASE_URL", "") or self.raw.get("agent", {}).get("base_url", "")
+
+    @property
+    def agent_timeout(self) -> int:
+        env = os.getenv("KB_AGENT_TIMEOUT", "")
+        if env:
+            try:
+                return int(env)
+            except ValueError:
+                pass
+        return self.raw.get("agent", {}).get("subagent_timeout", 900)
+
+    @property
+    def agent_max_concurrent(self) -> int:
+        env = os.getenv("KB_AGENT_MAX_CONCURRENT", "")
+        if env:
+            try:
+                return int(env)
+            except ValueError:
+                pass
+        return self.raw.get("agent", {}).get("max_concurrent_subagents", 5)
+
+    @property
+    def agent_backend(self) -> str:
+        """Determine execution backend: 'delegated' / 'dry-run' / 'api'."""
+        model = self.agent_model
+        if model == "delegated":
+            return "delegated"
+        if model == "dry-run":
+            return "dry-run"
+        return "api"
+
+    # --- Maven ---
+
+    def maven_auth(self, kb_name: str | None = None) -> str:
+        env = os.getenv("KB_MAVEN_AUTH", "")
+        if env:
+            return env
+        if kb_name:
+            kb = self.get_kb(kb_name)
+            auth = kb.get("maven_repo", {}).get("auth", "")
+            if auth:
+                return auth
+        for kb in self.raw.get("knowledge_bases", {}).values():
+            auth = kb.get("maven_repo", {}).get("auth", "")
+            if auth:
+                return auth
+        return ""
+
+    # --- ChromaDB ---
+
+    def chroma_dir(self, kb_name: str | None = None) -> Path:
+        if kb_name:
+            kb = self.get_kb(kb_name)
+            return Path(kb["knowledge_dir"]).parent / "chroma_db"
+        for kb in self.raw.get("knowledge_bases", {}).values():
+            return Path(kb["knowledge_dir"]).parent / "chroma_db"
+        return Path("./chroma_db")
+
+
+# ---------------------------------------------------------------------------
+# Loading functions
+# ---------------------------------------------------------------------------
+
+
+def find_config(start_dir: Path | None = None) -> Path:
+    """Search upward for kb-project.yaml. Raises FileNotFoundError if not found."""
+    current = (start_dir or Path.cwd()).resolve()
+    while True:
+        candidate = current / "kb-project.yaml"
+        if candidate.exists():
+            return candidate
+        parent = current.parent
+        if parent == current:
+            raise FileNotFoundError(f"kb-project.yaml not found (searched from {start_dir or Path.cwd()})")
+        current = parent
+
+
+def load_config(config_path: Path | None = None) -> Config:
+    """Load, validate, and resolve paths in kb-project.yaml. Returns Config instance."""
+    if config_path is None:
+        config_path = find_config()
+    config_path = Path(config_path)
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file does not exist: {config_path}")
+
+    with open(config_path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f)
+
+    errors = validate_config(raw)
+    if errors:
+        msg = "\n".join(f"  - {e}" for e in errors)
+        raise ValueError(f"Config validation failed:\n{msg}")
+
+    resolved = _resolve_paths(raw, config_path.parent)
+    return Config(raw=resolved, config_path=config_path)
+
+
+def validate_config(config: dict) -> list[str]:
+    """Validate configuration. Returns list of error strings (empty = valid)."""
+    errors: list[str] = []
+    if config.get("version") != 1:
+        errors.append("version must be 1")
+
+    kbs = config.get("knowledge_bases", {})
+    if not kbs:
+        errors.append("knowledge_bases must contain at least one entry")
+        return errors
+
+    collections: dict[str, str] = {}
+    for kb_name, kb_cfg in kbs.items():
+        coll = kb_cfg.get("collection")
+        if not coll:
+            errors.append(f"knowledge_bases.{kb_name}.collection is required")
+        elif coll in collections:
+            errors.append(f"collection '{coll}' duplicated: {collections[coll]} and {kb_name}")
+        else:
+            collections[coll] = kb_name
+
+        source = kb_cfg.get("source", {})
+        structure = source.get("structure")
+        if structure not in ("multi-repo", "monorepo"):
+            errors.append(f"knowledge_bases.{kb_name}.source.structure must be multi-repo or monorepo")
+
+        if not kb_cfg.get("preset"):
+            errors.append(f"knowledge_bases.{kb_name}.preset is required")
+
+    return errors
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _resolve_paths(config: dict, base_dir: Path) -> dict:
+    """Resolve relative paths to absolute. Returns a deep copy."""
+    config = copy.deepcopy(config)
+    base_dir = base_dir.resolve()
+
+    for _kb_name, kb_cfg in config.get("knowledge_bases", {}).items():
+        if "knowledge_dir" in kb_cfg:
+            kb_cfg["knowledge_dir"] = str(_resolve(kb_cfg["knowledge_dir"], base_dir))
+        source = kb_cfg.get("source", {})
+        if "cache_dir" in source:
+            source["cache_dir"] = str(_resolve(source["cache_dir"], base_dir))
+        if source.get("structure") == "multi-repo":
+            for repo in source.get("repos", []):
+                if "local" in repo:
+                    repo["local"] = str(_resolve(repo["local"], base_dir))
+    return config
+
+
+def _resolve(path_str: str, base_dir: Path) -> Path:
+    """Resolve a single path relative to base_dir."""
+    p = Path(path_str).expanduser()
+    if not p.is_absolute():
+        p = (base_dir / p).resolve()
+    return p

core/docs/__init__.py

@@ -0,0 +1,7 @@
+"""core.docs — shared document generation.
+
+Generates cross-module summary documents in _shared/ directory:
+- project-overview.md
+- cross-module-calls.md
+- error-codes.md (if applicable)
+"""