source-kb 0.2.2__py3-none-any.whl

core/paths.py

@@ -0,0 +1,141 @@
+"""Canonical path resolution for knowledge base artifacts.
+
+All modules use this single source of truth for file/directory paths.
+No hard-coded path strings elsewhere in the codebase.
+
+Directory structure:
+    knowledge/{kb}/{module}/
+    ├── .meta/
+    │   ├── skeleton/       # Skeleton files (JSON)
+    │   ├── file-lists/     # File classification lists
+    │   ├── prompts/        # Rendered sub-agent prompts
+    │   └── progress/       # Progress tracking
+    └── *.md                # Final documents
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+META_DIR = ".meta"
+
+
+# ─── Directory paths ────────────────────────────────────────
+
+def meta_dir(module_dir: Path) -> Path:
+    return module_dir / META_DIR
+
+
+def skeleton_dir(module_dir: Path) -> Path:
+    return meta_dir(module_dir) / "skeleton"
+
+
+def file_list_dir(module_dir: Path) -> Path:
+    return meta_dir(module_dir) / "file-lists"
+
+
+def prompts_dir(module_dir: Path) -> Path:
+    return meta_dir(module_dir) / "prompts"
+
+
+def progress_dir(module_dir: Path) -> Path:
+    return meta_dir(module_dir) / "progress"
+
+
+# ─── File paths ─────────────────────────────────────────────
+
+def skeleton_path(module_dir: Path) -> Path:
+    return skeleton_dir(module_dir) / "skeleton.json"
+
+
+def skeleton_shards_dir(module_dir: Path) -> Path:
+    return skeleton_dir(module_dir) / "shards"
+
+
+def skeleton_summary_path(module_dir: Path) -> Path:
+    return skeleton_dir(module_dir) / "summary.json"
+
+
+def skeleton_stats_path(module_dir: Path) -> Path:
+    return skeleton_dir(module_dir) / "stats.json"
+
+
+def file_list_path(module_dir: Path, doc_type: str) -> Path:
+    return file_list_dir(module_dir) / f"{doc_type}.txt"
+
+
+def prompt_path(module_dir: Path, doc_type: str) -> Path:
+    return prompts_dir(module_dir) / f"{doc_type}.md"
+
+
+def progress_path(module_dir: Path, doc_name: str) -> Path:
+    return progress_dir(module_dir) / doc_name
+
+
+def progress_pid_path(module_dir: Path, doc_name: str) -> Path:
+    return progress_dir(module_dir) / f"{doc_name}.hb-pid"
+
+
+# ─── Shard paths (unified naming for both modes) ───────────
+
+def shard_doc_path(module_dir: Path, doc_basename: str, shard_name: str) -> Path:
+    """Output path for a shard document: {module_dir}/{doc_basename}-{shard_name}.md"""
+    return module_dir / f"{doc_basename}-{shard_name}.md"
+
+
+def shard_file_list_path(module_dir: Path, doc_type: str, shard_name: str) -> Path:
+    """File list path for a shard: .meta/file-lists/{doc_type}-{shard_name}.txt"""
+    return file_list_dir(module_dir) / f"{doc_type}-{shard_name}.txt"
+
+
+# ─── Resolution (prefer new paths, fallback to legacy) ─────
+
+def resolve_skeleton(module_dir: Path) -> Path | None:
+    """Resolve skeleton file or shards directory. Returns None if not found."""
+    new = skeleton_path(module_dir)
+    if new.exists():
+        return new
+    new_shards = skeleton_shards_dir(module_dir)
+    if new_shards.is_dir():
+        return new_shards
+    # Legacy fallback
+    for legacy in (module_dir / ".skeleton.json", module_dir / ".skeleton"):
+        if legacy.exists():
+            return legacy
+    return None
+
+
+def resolve_skeleton_summary(module_dir: Path) -> Path | None:
+    """Resolve skeleton summary file."""
+    new = skeleton_summary_path(module_dir)
+    if new.exists():
+        return new
+    old = module_dir / ".skeleton-summary.json"
+    if old.exists():
+        return old
+    return None
+
+
+def resolve_file_list(module_dir: Path, doc_type: str) -> Path | None:
+    """Resolve file list for a doc type."""
+    new = file_list_path(module_dir, doc_type)
+    if new.exists():
+        return new
+    # Legacy fallback
+    for name in (f".file-list-{doc_type}.txt", f".file-list-{doc_type.split('-')[0]}.txt"):
+        old = module_dir / name
+        if old.exists():
+            return old
+    return None
+
+
+def resolve_progress(module_dir: Path, doc_name: str) -> Path | None:
+    """Resolve progress file. Returns None if not found."""
+    p = progress_path(module_dir, doc_name)
+    return p if p.exists() else None
+
+
+def ensure_dir(path: Path) -> Path:
+    """Create directory if it doesn't exist. Returns the path."""
+    path.mkdir(parents=True, exist_ok=True)
+    return path

core/preset.py

@@ -0,0 +1,237 @@
+"""Preset configuration loading and file classification.
+
+Loads language presets (java-spring, generic, etc.) and provides rule-based
+file classification. All classification rules come from preset.yaml config.
+
+Public API:
+    load_preset(name) → dict
+    classify_file(preset, file_path, skeleton_entry) → list[str]
+    get_doc_filename(preset, doc_type) → str
+    get_affected_docs(preset, categories) → set[str]
+    get_doc_type_mapping(preset) → dict
+    CompiledClassifier(preset) — optimized batch classifier
+"""
+
+from __future__ import annotations
+
+import copy
+import fnmatch
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+_BUILTIN_PRESETS_DIR = Path(__file__).resolve().parent.parent / "presets"
+PRESETS_DIR = _BUILTIN_PRESETS_DIR  # backward compat
+
+_preset_cache: dict[str, dict] = {}
+_preset_lock = __import__("threading").Lock()
+
+
+# ---------------------------------------------------------------------------
+# Preset search path
+# ---------------------------------------------------------------------------
+
+
+def _get_preset_search_dirs() -> list[Path]:
+    """Return preset directories in priority order (first match wins).
+
+    Search order:
+    1. SOURCE_KB_PRESETS_DIR env var (explicit override)
+    2. ./presets/ relative to cwd (project-local customization)
+    3. Built-in presets bundled with the package (fallback)
+    """
+    dirs: list[Path] = []
+
+    env_dir = os.environ.get("SOURCE_KB_PRESETS_DIR")
+    if env_dir:
+        p = Path(env_dir)
+        if p.is_dir():
+            dirs.append(p)
+
+    local_dir = Path.cwd() / "presets"
+    if local_dir.is_dir() and local_dir.resolve() != _BUILTIN_PRESETS_DIR.resolve():
+        dirs.append(local_dir)
+
+    dirs.append(_BUILTIN_PRESETS_DIR)
+    return dirs
+
+
+def _find_preset_dir(preset_name: str) -> Path | None:
+    """Find the first directory containing the named preset."""
+    for base in _get_preset_search_dirs():
+        candidate = base / preset_name / "preset.yaml"
+        if candidate.exists():
+            return base / preset_name
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Preset loading
+# ---------------------------------------------------------------------------
+
+
+def load_preset(preset_name: str) -> dict:
+    """Load preset config (with custom_rules + doc_types merge). Raises FileNotFoundError.
+
+    Search order: SOURCE_KB_PRESETS_DIR > ./presets/ > built-in presets.
+    Results are cached per preset_name. Returns a deep copy so callers can mutate safely.
+    """
+    with _preset_lock:
+        if preset_name in _preset_cache:
+            return copy.deepcopy(_preset_cache[preset_name])
+
+    _validate_name(preset_name)
+    preset_dir = _find_preset_dir(preset_name)
+    if preset_dir is None:
+        raise FileNotFoundError(
+            f"Preset '{preset_name}' not found. Searched: {[str(d) for d in _get_preset_search_dirs()]}"
+        )
+
+    preset_path = preset_dir / "preset.yaml"
+    with open(preset_path, encoding="utf-8") as f:
+        base = yaml.safe_load(f)
+
+    # Merge custom_rules.yaml if exists
+    custom_path = preset_dir / "custom_rules.yaml"
+    if custom_path.exists():
+        with open(custom_path, encoding="utf-8") as f:
+            custom = yaml.safe_load(f) or {}
+        base = _merge_custom_rules(base, custom)
+
+    # Merge doc_types.yaml if exists (doc_type definitions, split config, limits, parsers)
+    doc_types_path = preset_dir / "doc_types.yaml"
+    if doc_types_path.exists():
+        with open(doc_types_path, encoding="utf-8") as f:
+            doc_types_data = yaml.safe_load(f) or {}
+        # Merge top-level keys (doc_types, split, limits, parsers) into base
+        for key in ("doc_types", "split", "limits", "parsers"):
+            if key in doc_types_data:
+                base[key] = doc_types_data[key]
+
+    base["_preset_dir"] = str(preset_dir)
+
+    with _preset_lock:
+        _preset_cache[preset_name] = base
+    return copy.deepcopy(base)
+
+
+def find_preset_template(preset_name: str, template_name: str) -> Path | None:
+    """Find a template file within a preset's templates/ directory.
+
+    Uses the same search path as load_preset: env > local > built-in.
+    """
+    preset_dir = _find_preset_dir(preset_name)
+    if preset_dir is None:
+        return None
+    candidate = preset_dir / "templates" / template_name
+    if candidate.exists():
+        return candidate
+    return None
+
+
+def _validate_name(name: str):
+    import re
+    if not name or not re.match(r"^[a-zA-Z0-9][a-zA-Z0-9_-]*$", name):
+        raise ValueError(f"Invalid preset name: {name!r}")
+
+
+def _merge_custom_rules(base: dict, custom: dict) -> dict:
+    """Merge custom_rules into base preset (append patterns, add categories)."""
+    merged = copy.deepcopy(base)
+    custom_cls = custom.get("file_classification", {})
+    if not custom_cls:
+        return merged
+
+    base_cls = merged.setdefault("file_classification", {})
+    for cat, cfg in custom_cls.items():
+        if not isinstance(cfg, dict):
+            continue
+        clean = {k: v for k, v in cfg.items() if not k.startswith("_")}
+        if cat in base_cls:
+            for key, value in clean.items():
+                if key.endswith("_append") and isinstance(value, list):
+                    real_key = key.removesuffix("_append")
+                    existing = base_cls[cat].get(real_key, [])
+                    if isinstance(existing, list):
+                        for item in value:
+                            if item not in existing:
+                                existing.append(item)
+                        base_cls[cat][real_key] = existing
+        else:
+            base_cls[cat] = clean
+    return merged
+
+
+# ---------------------------------------------------------------------------
+# Doc type mapping
+# ---------------------------------------------------------------------------
+
+
+def get_doc_filename(preset: dict, doc_type: str, *, strict: bool = False) -> str:
+    """Get the output filename for a doc type key."""
+    doc_types = preset.get("doc_types", {})
+    if doc_type in doc_types:
+        return doc_types[doc_type].get("filename", f"{doc_type}.md")
+    if strict:
+        raise KeyError(f"Unknown doc-type '{doc_type}'. Valid: {sorted(doc_types.keys())}")
+    return f"{doc_type}.md"
+
+
+def get_doc_type_mapping(preset: dict) -> dict[str, str]:
+    """Return full doc-type → filename mapping (built from doc_types config)."""
+    doc_types = preset.get("doc_types", {})
+    if doc_types:
+        return {dt: cfg.get("filename", f"{dt}.md") for dt, cfg in doc_types.items()}
+    return {}
+
+
+def get_affected_docs(preset: dict, categories: list[str]) -> set[str]:
+    """Given category names, return affected document filenames."""
+    classification = preset.get("file_classification", {})
+    docs: set[str] = set()
+    for cat in categories:
+        cfg = classification.get(cat, {})
+        docs.update(cfg.get("affects", []))
+    return docs
+
+
+def get_file_classification(preset: dict) -> dict:
+    """Return the file_classification section."""
+    return dict(preset.get("file_classification", {}))
+
+
+# ---------------------------------------------------------------------------
+# File classification (delegated to preset_classify.py)
+# ---------------------------------------------------------------------------
+
+from core.preset_classify import classify_file  # noqa: E402, F401
+
+
+# ---------------------------------------------------------------------------
+# Utility accessors (delegated to preset_accessors.py)
+# ---------------------------------------------------------------------------
+
+from core.preset_accessors import (  # noqa: E402, F401
+    get_coverage_skip_patterns,
+    get_generation_order,
+    get_doc_types,
+    get_doc_type_config,
+    get_template_path,
+    get_split_override,
+    get_ownership_keywords,
+    get_global_view_types,
+    get_limits,
+    get_search_routing,
+    get_dependency_chain,
+    get_dedup_rules,
+    get_batch_plan,
+)
+
+
+# ---------------------------------------------------------------------------
+# Profile resolution and show-config (delegated to preset_profile.py)
+# ---------------------------------------------------------------------------
+
+from core.preset_profile import resolve_profile, show_config  # noqa: E402, F401

core/preset_accessors.py

@@ -0,0 +1,202 @@
+"""Preset utility accessors — query preset config for various subsystems.
+
+All functions take a loaded preset dict and return derived configuration.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def get_coverage_skip_patterns(preset: dict) -> list[str]:
+    return list(preset.get("coverage_skip_patterns", []))
+
+
+def get_generation_order(preset: dict) -> list[list[str]]:
+    """Return batch generation order derived from doc_types config.
+
+    Groups doc_types by their 'batch' field, sorted ascending.
+    Each batch is a list of doc_type keys that can be generated in parallel.
+    """
+    doc_types = preset.get("doc_types", {})
+    if not doc_types:
+        return list(preset.get("generation_order", [["source-tree-analysis"]]))
+
+    batches: dict[int, list[str]] = {}
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        batch_num = dt_config.get("batch", 99)
+        batches.setdefault(batch_num, []).append(dt_key)
+
+    return [batches[b] for b in sorted(batches.keys())]
+
+
+def get_doc_types(preset: dict) -> dict[str, dict]:
+    """Return the full doc_types configuration section."""
+    return dict(preset.get("doc_types", {}))
+
+
+def get_doc_type_config(preset: dict, doc_type: str) -> dict:
+    """Get config for a specific doc_type. Returns empty dict if not found."""
+    doc_types = preset.get("doc_types", {})
+    return doc_types.get(doc_type, {})
+
+
+def get_template_path(preset: dict, doc_type: str, preset_name: str) -> str | None:
+    """Get the template filename for a doc_type from preset config.
+
+    Returns relative path like 'subagent-business.md', or None if not configured.
+    """
+    dt_config = get_doc_type_config(preset, doc_type)
+    return dt_config.get("template")
+
+
+def get_split_override(preset: dict, doc_type: str, mode: str) -> dict:
+    """Get per-doc-type split threshold overrides."""
+    dt_config = get_doc_type_config(preset, doc_type)
+    overrides = dt_config.get("split_override", {})
+    return overrides.get(mode, {})
+
+
+def get_ownership_keywords(preset: dict) -> dict[str, list[str]]:
+    """Get doc_type -> ownership keywords mapping (for duplicate detection)."""
+    doc_types = preset.get("doc_types", {})
+    result: dict[str, list[str]] = {}
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        keywords = dt_config.get("owns_keywords", [])
+        if keywords:
+            filename = dt_config.get("filename", f"{dt_key}.md")
+            result[filename] = keywords
+    return result
+
+
+def get_global_view_types(preset: dict) -> frozenset[str]:
+    """Return all doc_type keys marked as global_view."""
+    doc_types = preset.get("doc_types", {})
+
+    global_keys: set[str] = set()
+    for dt_key, dt_config in doc_types.items():
+        if isinstance(dt_config, dict) and dt_config.get("global_view", False):
+            global_keys.add(dt_key)
+
+    if not global_keys and not doc_types:
+        return frozenset({"source-tree-analysis", "index"})
+
+    return frozenset(global_keys)
+
+
+def get_limits(preset: dict, project_config: dict | None = None) -> dict[str, int]:
+    """Get configurable limits (timeouts, size thresholds, etc.).
+
+    Resolution order: project_config (kb-project.yaml) > preset (doc_types.yaml) > defaults.
+    """
+    defaults = {
+        "min_doc_size_bytes": 500,
+        "max_source_inline_bytes": 300000,
+        "max_skeleton_inline_bytes": 50000,
+        "max_output_tokens": 8192,
+        "prior_docs_max_chars": 2000,
+        "shard_context_max_chars": 1500,
+        "spawn_timeout_default": 900,
+        "heartbeat_interval": 30,
+        "max_retries": 2,
+    }
+    configured = preset.get("limits", {})
+    result = {**defaults, **configured}
+    if project_config:
+        project_limits = project_config.get("limits", {})
+        result = {**result, **project_limits}
+    return result
+
+
+def get_search_routing(preset: dict) -> dict[str, list[str]]:
+    """Get filename -> search keywords mapping for question routing."""
+    doc_types = preset.get("doc_types", {})
+    result: dict[str, list[str]] = {}
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        filename = dt_config.get("filename", f"{dt_key}.md")
+        keywords = dt_config.get("search_keywords", [])
+        if not keywords:
+            keywords = [dt_key.replace("-", " ")]
+        result[filename] = keywords
+    return result
+
+
+def get_dependency_chain(preset: dict) -> list[str]:
+    """Return doc_type keys in dependency-safe generation/update order (topological sort)."""
+    doc_types = preset.get("doc_types", {})
+    if not doc_types:
+        return []
+
+    deps: dict[str, set[str]] = {}
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        depends_on = dt_config.get("depends_on", [])
+        deps[dt_key] = set(depends_on) & set(doc_types.keys())
+
+    in_degree: dict[str, int] = {k: len(v) for k, v in deps.items()}
+    queue: list[str] = sorted(
+        [k for k, d in in_degree.items() if d == 0],
+        key=lambda k: (doc_types.get(k, {}).get("batch", 99), k),
+    )
+    result: list[str] = []
+
+    while queue:
+        node = queue.pop(0)
+        result.append(node)
+        for k, dep_set in deps.items():
+            if node in dep_set:
+                dep_set.discard(node)
+                in_degree[k] -= 1
+                if in_degree[k] == 0:
+                    queue.append(k)
+                    queue.sort(key=lambda x: (doc_types.get(x, {}).get("batch", 99), x))
+
+    for k in deps:
+        if k not in result:
+            result.append(k)
+
+    return result
+
+
+def get_dedup_rules(preset: dict) -> dict[str, list[str]]:
+    """Get filename -> dedup detection patterns."""
+    doc_types = preset.get("doc_types", {})
+    result: dict[str, list[str]] = {}
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        patterns = list(dt_config.get("dedup_patterns", []))
+        owns = dt_config.get("owns_keywords", [])
+        if patterns or owns:
+            filename = dt_config.get("filename", f"{dt_key}.md")
+            result[filename] = patterns + owns
+    return result
+
+
+def get_batch_plan(preset: dict) -> dict[int, list[dict]]:
+    """Get batch_num -> list of doc_type info dicts for generation planning."""
+    doc_types = preset.get("doc_types", {})
+    result: dict[int, list[dict]] = {}
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        batch_num = dt_config.get("batch", 99)
+        entry = {
+            "name": dt_key,
+            "filename": dt_config.get("filename", f"{dt_key}.md"),
+            "conditional": dt_config.get("conditional", False),
+            "depends_on": dt_config.get("depends_on", []),
+            "global_view": dt_config.get("global_view", False),
+            "template": dt_config.get("template"),
+        }
+        result.setdefault(batch_num, []).append(entry)
+    for batch_num in result:
+        result[batch_num].sort(key=lambda x: x["name"])
+    return result