source-kb 0.2.2__py3-none-any.whl

core/prompt/variables.py

@@ -0,0 +1,339 @@
+"""Prompt variables computation — transforms content into template-ready strings.
+
+Computes all template variables needed for prompt rendering. This module is
+responsible for *formatting* (converting raw data into template-usable strings),
+while content.py handles *reading* (I/O from files).
+
+No CLI I/O, no LLM calls.
+
+Usage:
+    from core.prompt.variables import PromptVariables, compute_variables
+
+    variables = compute_variables(
+        module_dir=module_dir,
+        doc_type="business-logic",
+        config=config,
+        kb_name="point",
+        module_name="point-base",
+    )
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import PromptAssembler
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# PromptVariables dataclass
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PromptVariables:
+    """All variables needed for prompt template rendering."""
+
+    module_name: str = ""
+    module_description: str = ""
+    module_dir: str = ""
+    doc_type: str = ""
+    source_cache_path: str = ""
+    output_path: str = ""
+    file_list: str = ""
+    source_content: str = ""
+    skeleton_content: str = ""
+    high_methods: str = ""
+    generated_docs: str = ""
+    sibling_modules: str = ""
+    global_metadata: str = ""
+    prior_docs_context: str = ""
+    shard_context: str = ""
+    branch: str = "main"
+    module_type: str = "service"
+    extras: dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, str]:
+        """Convert to flat dict for template substitution."""
+        result = {
+            "module_name": self.module_name,
+            "module_description": self.module_description,
+            "module_dir": self.module_dir,
+            "doc_type": self.doc_type,
+            "source_cache_path": self.source_cache_path,
+            "output_path": self.output_path,
+            "file_list": self.file_list,
+            "source_content": self.source_content,
+            "skeleton_content": self.skeleton_content,
+            "high_methods": self.high_methods,
+            "generated_docs": self.generated_docs,
+            "generated_docs_files": self.generated_docs,
+            "sibling_modules": self.sibling_modules,
+            "global_metadata": self.global_metadata,
+            "prior_docs_context": self.prior_docs_context,
+            "shard_context": self.shard_context,
+            "branch": self.branch,
+            "module_type": self.module_type,
+        }
+        result.update(self.extras)
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Variable computation
+# ---------------------------------------------------------------------------
+
+
+def compute_variables(
+    module_dir: Path,
+    doc_type: str,
+    config: dict[str, Any],
+    kb_name: str,
+    module_name: str,
+    source_cache: Path | None = None,
+    preset: dict[str, Any] | None = None,
+    extras: dict[str, str] | None = None,
+    shard_context: str = "",
+) -> PromptVariables:
+    """Compute all prompt variables from config and file system state.
+
+    Args:
+        module_dir: Module knowledge directory
+        doc_type: Document type key
+        config: Full kb-project.yaml config dict
+        kb_name: Knowledge base name
+        module_name: Module name
+        source_cache: Source code cache directory (auto-resolved if None)
+        preset: Preset config dict (optional)
+        extras: Additional variables to inject
+        shard_context: Pre-computed shard context string
+
+    Returns:
+        PromptVariables with all fields populated
+    """
+    from core.prompt.content import (
+        compute_high_methods,
+        scan_generated_docs,
+        compute_sibling_modules,
+        build_prior_docs_context,
+    )
+    from core.skeleton.metadata import load_pregenerated
+
+    kb_config = config["knowledge_bases"][kb_name]
+    repo_info = _find_repo(config, kb_name, module_name)
+
+    # Resolve source_cache if not provided
+    if source_cache is None:
+        source = kb_config.get("source", {})
+        base_dir = Path(config.get("_config_dir", ".")).resolve()
+        cache_dir = Path(source.get("cache_dir", "./.source-cache"))
+        if not cache_dir.is_absolute():
+            cache_dir = (base_dir / cache_dir).resolve()
+        if source.get("structure") == "monorepo":
+            repo_name = source.get("repo_name", "repo")
+            module_path = repo_info.get("path", module_name)
+            source_cache = cache_dir / repo_name / module_path
+        else:
+            source_cache = cache_dir / module_name
+
+    variables = PromptVariables(
+        module_name=module_name,
+        module_description=repo_info.get("description", f"{module_name} module"),
+        module_dir=str(module_dir).replace("\\", "/"),
+        doc_type=doc_type,
+        source_cache_path=str(source_cache).replace("\\", "/"),
+        output_path=str(module_dir).replace("\\", "/"),
+        high_methods=compute_high_methods(module_dir),
+        generated_docs=scan_generated_docs(module_dir),
+        sibling_modules=compute_sibling_modules(config, kb_name, module_name),
+        global_metadata=load_pregenerated(module_dir),
+        prior_docs_context=build_prior_docs_context(module_dir, doc_type, preset=preset),
+        shard_context=shard_context,
+        branch=repo_info.get("branch", "main"),
+        module_type=repo_info.get("type", "service"),
+        extras=extras or {},
+    )
+
+    return variables
+
+
+def format_file_list_size(file_list_content: str) -> str:
+    """Format file list content with size statistics.
+
+    Args:
+        file_list_content: Raw file list text
+
+    Returns:
+        Formatted string with file count appended
+    """
+    if not file_list_content or file_list_content.startswith("("):
+        return file_list_content
+
+    lines = [l for l in file_list_content.splitlines() if l.strip()]
+    count = len(lines)
+    return f"{file_list_content}\n\n Total: {count} files"
+
+
+def format_high_methods_summary(high_methods: str, max_items: int = 30) -> str:
+    """Truncate high methods list if too long.
+
+    Args:
+        high_methods: Raw high methods text
+        max_items: Maximum number of items to show
+
+    Returns:
+        Truncated string with count note if needed
+    """
+    if not high_methods or high_methods.startswith("("):
+        return high_methods
+
+    lines = high_methods.splitlines()
+    if len(lines) <= max_items:
+        return high_methods
+
+    truncated = "\n".join(lines[:max_items])
+    return f"{truncated}\n\n... {len(lines) - max_items} more high-complexity methods"
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _find_repo(config: dict[str, Any], kb_name: str, module_name: str) -> dict[str, Any]:
+    """Find repo/module config entry by name."""
+    kb = config["knowledge_bases"][kb_name]
+    source = kb["source"]
+    if source.get("structure") == "monorepo":
+        for mod in source.get("modules", []):
+            if mod["name"] == module_name:
+                return mod
+    else:
+        for repo in source.get("repos", []):
+            if repo["name"] == module_name:
+                return repo
+    return {"name": module_name}
+
+
+# ---------------------------------------------------------------------------
+# ReferencePromptAssembler — Agent mode (paths only, no inline content)
+# ---------------------------------------------------------------------------
+
+
+class ReferencePromptAssembler(PromptAssembler):
+    """Emits file path references instead of inlining content.
+
+    Used in Agent/readwrite mode where sub-agents can read files themselves.
+    Implements the PromptAssembler interface from core.interfaces.
+
+    Follows the old engine/skeleton/render_prompt.py three-way branching:
+    1. Global_View_Doc (from preset doc_types.yaml global_view: true) → skeleton summary reference
+    2. Agent mode (readwrite) + Regular_Doc → file-list path reference
+    3. CLI mode (output-only) + Regular_Doc → inline file content (handled by InlinePromptAssembler)
+    """
+
+    def __init__(self, project_root: Path | None = None, preset: dict | None = None):
+        self._project_root = project_root or Path(".").resolve()
+        self._global_view_types = self._load_global_view_types(preset)
+
+    @staticmethod
+    def _load_global_view_types(preset: dict | None) -> set[str]:
+        """Load global_view doc types from preset config dynamically."""
+        if not preset:
+            return {"source-tree-analysis", "index"}
+        from core.preset import get_global_view_types
+        return set(get_global_view_types(preset))
+
+    def _relative(self, path: Path) -> str:
+        """Convert absolute path to project-relative path string."""
+        try:
+            return str(path.relative_to(self._project_root)).replace("\\", "/")
+        except ValueError:
+            return str(path).replace("\\", "/")
+
+    def resolve_file_list(self, module_dir: Path, doc_type: str,
+                         file_list_override: str | None = None) -> str:
+        """Return file list as relative path reference — sub-agent reads it.
+
+        For Global_View_Doc types, returns skeleton summary reference instead.
+
+        Args:
+            module_dir: Module documentation directory
+            doc_type: Document type key
+            file_list_override: Optional override path (for shard-specific file lists)
+        """
+        if doc_type in self._global_view_types:
+            return self._skeleton_summary_reference(module_dir)
+
+        # Check override first (shard-specific file list)
+        if file_list_override:
+            override_path = Path(file_list_override)
+            if override_path.exists():
+                content = override_path.read_text(encoding="utf-8").strip()
+                lines = [l.strip() for l in content.splitlines()
+                         if l.strip() and not l.strip().startswith("#")]
+                rel_path = self._relative(override_path)
+                return (
+                    f"File list path: `{rel_path}`\n"
+                    f"Total files: {len(lines)} source files\n\n"
+                    f"Please read the file list above to get source file paths, then read source files grouped by package."
+                )
+
+        # Regular doc: file-list path reference
+        from core.paths import resolve_file_list
+        fl = resolve_file_list(module_dir, doc_type)
+        if fl is None or not fl.exists():
+            return "File list does not exist, please run `source-kb file-list` first to generate it"
+        content = fl.read_text(encoding="utf-8").strip()
+        if not content:
+            return "(No matching files)"
+        lines = [l.strip() for l in content.splitlines()
+                 if l.strip() and not l.strip().startswith("#")]
+        rel_path = self._relative(fl)
+        return (
+            f"File list path: `{rel_path}`\n"
+            f"Total files: {len(lines)} source files\n\n"
+            f"Please read the file list above to get source file paths, then read source files grouped by package."
+        )
+
+    def _skeleton_summary_reference(self, module_dir: Path) -> str:
+        """Return skeleton summary path + source cache hint for Global_View_Doc types."""
+        from core.paths import resolve_skeleton_summary
+        summary = resolve_skeleton_summary(module_dir)
+        if summary and summary.exists():
+            rel_path = self._relative(summary)
+            # Also provide source_cache path for optional deep-dive
+            source_cache_hint = self._relative(module_dir).replace("knowledge/", ".source-cache/").rsplit("/", 1)[0]
+            return (
+                f"Skeleton summary path: `{rel_path}`\n\n"
+                f"This is a global view document type, generated primarily from the skeleton summary — no need to read source files individually.\n"
+                f"If you need implementation details of a specific class, source code is located in: `.source-cache/` under the corresponding module directory."
+            )
+        return (
+            "Skeleton summary file does not exist, please run "
+            "`source-kb extract --repo ... --preset ... --output ... --summary` first to generate it"
+        )
+
+    def resolve_source_content(self, module_dir: Path, doc_type: str, source_cache: Path) -> str:
+        """Return empty — sub-agent reads files itself via file-list."""
+        return ""
+
+    def resolve_skeleton_content(self, module_dir: Path) -> str:
+        """Return skeleton path reference."""
+        from core.paths import resolve_skeleton, resolve_skeleton_summary
+        summary = resolve_skeleton_summary(module_dir)
+        if summary and summary.exists():
+            return f"Skeleton summary path: `{self._relative(summary)}`\n(Please use the Read tool to read it)"
+        resolved = resolve_skeleton(module_dir)
+        if resolved and resolved.exists():
+            return f"Skeleton path: `{self._relative(resolved)}`\n(Please use the Read tool to read it)"
+        return "(Skeleton does not exist)"
+
+    def should_append_source(self) -> bool:
+        """Agent mode does not append source — sub-agents read files themselves."""
+        return False

core/rag/__init__.py

@@ -0,0 +1,22 @@
+"""core.rag — vector index (chunking, embedding, retrieval).
+
+Provides document loading, chunking, index building, and retrieval.
+Embedding backend is configurable via Config (ollama, openai, dashscope, chromadb-default).
+"""
+
+from core.rag.loader import Document, load_documents
+from core.rag.chunker import Chunk, chunk_documents
+from core.rag.embedder import get_embeddings
+from core.rag.indexer import build_index, get_collection
+from core.rag.retriever import retrieve
+
+__all__ = [
+    "Document",
+    "load_documents",
+    "Chunk",
+    "chunk_documents",
+    "get_embeddings",
+    "build_index",
+    "get_collection",
+    "retrieve",
+]

core/rag/__main__.py

@@ -0,0 +1,136 @@
+"""CLI entry points for RAG operations (search, index, rebuild).
+
+Provides sub-commands for Agent mode:
+    python -m core.rag search --kb my-kb "query"
+    python -m core.rag index --kb my-kb
+    python -m core.rag rebuild --kb my-kb --module my-module --files a.md b.md
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+
+def cmd_search(args: argparse.Namespace) -> None:
+    """Search knowledge base."""
+    from core.config import load_config, find_config
+    from core.rag.retriever import retrieve
+
+    config_path = Path(args.config) if args.config else find_config()
+    config = load_config(config_path)
+
+    results = retrieve(args.query, config, kb_name=args.kb)
+    if not results:
+        print(f"No results for '{args.query}'")
+        print(json.dumps({"status": "ok", "matches": 0}, ensure_ascii=False), file=sys.stderr)
+        return
+
+    for i, r in enumerate(results, 1):
+        score = r["score"]
+        source = r["metadata"].get("source", "?")
+        section = r["metadata"].get("section", "")
+        header = f"[{i}] {source}"
+        if section:
+            header += f" > {section}"
+        print(f"{header}  (score: {score:.3f})")
+        print(f"    {r['text'][:200]}...")
+        print()
+
+    print(json.dumps({"status": "ok", "matches": len(results)}, ensure_ascii=False), file=sys.stderr)
+
+
+def cmd_index(args: argparse.Namespace) -> None:
+    """Build/rebuild full vector index."""
+    from core.config import load_config, find_config
+    from core.rag.loader import load_documents
+    from core.rag.chunker import chunk_documents
+    from core.rag.indexer import build_index
+
+    config_path = Path(args.config) if args.config else find_config()
+    config = load_config(config_path)
+    kb_cfg = config.get_kb(args.kb)
+    knowledge_dir = kb_cfg["knowledge_dir"]
+    collection_name = kb_cfg["collection"]
+
+    docs = load_documents(knowledge_dir)
+    if not docs:
+        print(f"No documents found in {knowledge_dir}")
+        print(json.dumps({"status": "error", "message": "no documents"}, ensure_ascii=False), file=sys.stderr)
+        return
+
+    chunks = chunk_documents(docs)
+    print(f"Indexing {len(docs)} docs, {len(chunks)} chunks...")
+    build_index(chunks, collection_name, config, kb_name=args.kb)
+    print(f"Index built: {len(chunks)} chunks -> collection '{collection_name}'")
+    print(json.dumps({"status": "ok", "docs": len(docs), "chunks": len(chunks),
+                      "collection": collection_name}, ensure_ascii=False), file=sys.stderr)
+
+
+def cmd_rebuild(args: argparse.Namespace) -> None:
+    """Incremental rebuild — re-index only specified files."""
+    from core.config import load_config, find_config
+    from core.rag.loader import load_documents
+    from core.rag.chunker import chunk_documents
+    from core.rag.indexer import build_index
+
+    config_path = Path(args.config) if args.config else find_config()
+    config = load_config(config_path)
+    kb_cfg = config.get_kb(args.kb)
+    knowledge_dir = Path(kb_cfg["knowledge_dir"])
+    collection_name = kb_cfg["collection"]
+
+    if args.module:
+        knowledge_dir = knowledge_dir / args.module
+
+    files = args.files or []
+    if files:
+        docs = load_documents(knowledge_dir, file_filter=files)
+    else:
+        docs = load_documents(knowledge_dir)
+
+    if not docs:
+        print(f"No documents to rebuild")
+        print(json.dumps({"status": "ok", "rebuilt": 0}, ensure_ascii=False), file=sys.stderr)
+        return
+
+    chunks = chunk_documents(docs)
+    print(f"Rebuilding {len(docs)} docs, {len(chunks)} chunks...")
+    build_index(chunks, collection_name, config, kb_name=args.kb, incremental=True)
+    print(f"Rebuilt: {len(chunks)} chunks -> collection '{collection_name}'")
+    print(json.dumps({"status": "ok", "docs": len(docs), "chunks": len(chunks)},
+                     ensure_ascii=False), file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="python -m core.rag", description="RAG tools")
+    parser.add_argument("--config", help="kb-project.yaml path (auto-detected)")
+    sub = parser.add_subparsers(dest="command")
+
+    p = sub.add_parser("search", help="Search knowledge base")
+    p.add_argument("--kb", required=True, help="Knowledge base name")
+    p.add_argument("query", help="Search query")
+
+    p = sub.add_parser("index", help="Build/rebuild full vector index")
+    p.add_argument("--kb", required=True, help="Knowledge base name")
+
+    p = sub.add_parser("rebuild", help="Incremental rebuild (specific files)")
+    p.add_argument("--kb", required=True, help="Knowledge base name")
+    p.add_argument("--module", help="Module name")
+    p.add_argument("--files", nargs="*", help="Specific files to rebuild")
+
+    args = parser.parse_args()
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    commands = {"search": cmd_search, "index": cmd_index, "rebuild": cmd_rebuild}
+    commands[args.command](args)
+
+
+if __name__ == "__main__":
+    main()