source-kb 0.2.2__py3-none-any.whl

engine/dedup.py

@@ -0,0 +1,106 @@
+"""LLM-based document deduplication — CLI-only post-merge optimization.
+
+Uses LLM to intelligently remove cross-document redundancy:
+- Internal dedup: repeated tables/paragraphs within a document
+- Cross-doc dedup: content that belongs in another document → replace with reference
+
+Only processes documents exceeding a configurable size threshold.
+Agent mode uses rule-based refinement (core/skeleton/merge.py) instead.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from pathlib import Path
+
+from core.interfaces import LlmRequest, LlmStrategy
+
+logger = logging.getLogger(__name__)
+
+# Configurable thresholds (should come from config in production)
+MIN_SIZE_KB = 15
+CHUNK_THRESHOLD_KB = 60
+
+SYSTEM_PROMPT = (
+    "You are a technical documentation editing expert. Your task is to deduplicate and streamline a knowledge base document.\n"
+    "Rules:\n"
+    "1. Internal dedup: if the same content appears multiple times, keep only the most complete version\n"
+    "2. Cross-document dedup: replace content that belongs in another document with a cross-reference `> See [xxx.md](./xxx.md)`\n"
+    "3. Preserve structure: keep heading hierarchy, do not add new content\n"
+    "4. Merge conflicting data: keep the more specific version\n"
+    "Output the optimized full markdown document content directly."
+)
+
+
+def dedup_document(
+    doc_path: Path,
+    sibling_docs: list[str],
+    strategy: LlmStrategy,
+) -> tuple[bool, float]:
+    """Deduplicate a single document using LLM.
+
+    Args:
+        doc_path: Path to the document to deduplicate
+        sibling_docs: Names of other docs in the same module (for cross-doc reference)
+        strategy: LLM execution strategy
+
+    Returns:
+        (changed: bool, saved_kb: float) — whether content changed and KB saved
+    """
+    content = doc_path.read_text(encoding="utf-8")
+    original_size = len(content)
+
+    if original_size < MIN_SIZE_KB * 1024:
+        return False, 0.0
+
+    user_prompt = _build_prompt(doc_path.name, content, sibling_docs)
+
+    resp = strategy.call(LlmRequest(
+        system=SYSTEM_PROMPT,
+        user=user_prompt,
+        max_tokens=16384,
+        temperature=0.1,
+    ))
+
+    if resp.status != "done" or not resp.content:
+        return False, 0.0
+
+    new_content = _strip_fence(resp.content)
+    new_size = len(new_content)
+
+    # Safety: reject if too short (lost content) or longer (added content)
+    if new_size < original_size * 0.3 or new_size >= original_size:
+        return False, 0.0
+
+    if not new_content.endswith("\n"):
+        new_content += "\n"
+    doc_path.write_text(new_content, encoding="utf-8")
+
+    saved_kb = (original_size - new_size) / 1024
+    logger.info("[dedup] %s: %.1fKB → %.1fKB (saved %.1fKB)",
+                doc_path.name, original_size / 1024, new_size / 1024, saved_kb)
+    return True, saved_kb
+
+
+def _build_prompt(doc_name: str, content: str, siblings: list[str]) -> str:
+    siblings_str = ", ".join(f"`{s}`" for s in siblings if s != doc_name)
+    return (
+        f"Filename: `{doc_name}`\n"
+        f"Sibling documents in same directory: {siblings_str}\n\n"
+        f"```markdown\n{content}\n```\n\n"
+        f"Output the deduplicated and optimized full document content."
+    )
+
+
+def _strip_fence(text: str) -> str:
+    """Remove markdown code fence wrapper from LLM output."""
+    content = text.strip()
+    for prefix in ("```markdown", "```md", "```"):
+        if content.startswith(prefix):
+            content = content[len(prefix):].strip()
+            break
+    if content.endswith("```"):
+        content = content[:-3].strip()
+    return content

engine/main.py

@@ -0,0 +1,211 @@
+"""CLI entry point — simplified commands with sensible defaults.
+
+Usage:
+    python -m cli.main init --kb my-kb
+    python -m cli.main sync --kb my-kb
+    python -m cli.main audit --kb my-kb
+    python -m cli.main search --kb my-kb "query"
+    python -m cli.main index --kb my-kb
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+from core import __version__, setup_logging
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+
+def cmd_init(args: argparse.Namespace) -> None:
+    """Run kb-init pipeline."""
+    config, ctx = _setup(args)
+    from engine.pipelines import build_init_pipeline
+    from engine.pipeline.recovery import load_checkpoint, should_skip_step
+
+    pipeline = build_init_pipeline()
+    if args.dry_run:
+        _print_plan(pipeline)
+        return
+
+    # Resume support
+    checkpoint = None
+    if args.resume:
+        checkpoint = load_checkpoint(ctx.knowledge_dir)
+        if checkpoint is None:
+            print("No checkpoint found — starting from scratch")
+        else:
+            print(f"Resuming from: {checkpoint.next_step}")
+
+    result = pipeline.execute(ctx, checkpoint=checkpoint)
+    _print_result(result)
+    if not result.passed:
+        sys.exit(1)
+
+
+def cmd_sync(args: argparse.Namespace) -> None:
+    """Run kb-sync pipeline."""
+    config, ctx = _setup(args)
+    from engine.pipelines import build_sync_pipeline
+
+    if getattr(args, "dry_run", False):
+        ctx.state["dry_run"] = True
+
+    pipeline = build_sync_pipeline()
+    result = pipeline.execute(ctx)
+    _print_result(result)
+    if not result.passed:
+        sys.exit(1)
+
+
+def cmd_audit(args: argparse.Namespace) -> None:
+    """Run kb-audit pipeline."""
+    config, ctx = _setup(args)
+    from engine.pipelines import build_audit_pipeline
+
+    if getattr(args, "dry_run", False):
+        ctx.state["dry_run"] = True
+    if getattr(args, "scope", None):
+        ctx.state["audit_scope"] = args.scope
+    if getattr(args, "force", False):
+        ctx.state["audit_force"] = True
+
+    pipeline = build_audit_pipeline()
+    result = pipeline.execute(ctx)
+    _print_result(result)
+    if not result.passed:
+        sys.exit(1)
+
+
+def cmd_search(args: argparse.Namespace) -> None:
+    """Search knowledge base."""
+    config, _ = _setup(args, need_ctx=False)
+    from core.rag.retriever import retrieve
+
+    results = retrieve(args.query, config, kb_name=args.kb)
+    if not results:
+        print(f"No results for '{args.query}'")
+        return
+    for i, r in enumerate(results, 1):
+        score = r["score"]
+        source = r["metadata"].get("source", "?")
+        section = r["metadata"].get("section", "")
+        header = f"[{i}] {source}"
+        if section:
+            header += f" > {section}"
+        print(f"{header}  (score: {score:.3f})")
+        print(f"    {r['text'][:200]}...")
+        print()
+
+
+def cmd_index(args: argparse.Namespace) -> None:
+    """Build/rebuild vector index."""
+    config, _ = _setup(args, need_ctx=False)
+    from core.rag.loader import load_documents
+    from core.rag.chunker import chunk_documents
+    from core.rag.indexer import build_index
+
+    kb_cfg = config.get_kb(args.kb)
+    knowledge_dir = kb_cfg["knowledge_dir"]
+    collection_name = kb_cfg["collection"]
+
+    docs = load_documents(knowledge_dir)
+    if not docs:
+        print(f"No documents found in {knowledge_dir}")
+        return
+
+    chunks = chunk_documents(docs)
+    print(f"Loaded {len(docs)} docs, {len(chunks)} chunks. Building index...")
+    build_index(chunks, collection_name, config, kb_name=args.kb)
+    print(f"Index built: {len(chunks)} chunks → collection '{collection_name}'")
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _setup(args: argparse.Namespace, need_ctx: bool = True):
+    """Load config and create pipeline context."""
+    from core.config import load_config, find_config
+    from core.interfaces import PipelineContext
+    from pathlib import Path
+
+    config_path = Path(args.config) if args.config else find_config()
+    config = load_config(config_path)
+
+    if not need_ctx:
+        return config, None
+
+    ctx = PipelineContext.from_config(config.raw, args.kb, module=getattr(args, "module", None))
+    return config, ctx
+
+
+def _print_plan(pipeline) -> None:
+    """Print pipeline steps without executing."""
+    print(f"[pipeline] {pipeline.name} — {len(pipeline.steps)} steps:")
+    for info in pipeline.describe():
+        cp = f"  (checkpoint={info['checkpoint']})" if info["checkpoint"] else ""
+        print(f"  {info['name']}{cp}")
+
+
+def _print_result(result) -> None:
+    """Print pipeline execution result."""
+    markers = {"ok": "+", "skipped": "-", "failed": "!", "delegated": "~"}
+    print(f"\n{result.summary()}")
+    for step_name, sr in result.step_results:
+        marker = markers.get(sr.status, "?")
+        print(f"  [{marker}] {step_name}: {sr.message}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="source-kb CLI")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument("--config", help="kb-project.yaml path (auto-detected)")
+    sub = parser.add_subparsers(dest="command")
+
+    # init
+    p_init = sub.add_parser("init", help="Initialize knowledge base")
+    p_init.add_argument("--kb", required=True)
+    p_init.add_argument("--module", default=None)
+    p_init.add_argument("--dry-run", action="store_true")
+    p_init.add_argument("--resume", action="store_true", help="Resume from last checkpoint")
+
+    # sync
+    p_sync = sub.add_parser("sync", help="Incremental sync")
+    p_sync.add_argument("--kb", required=True)
+    p_sync.add_argument("--module", default=None)
+    p_sync.add_argument("--dry-run", action="store_true", help="Preview changes without writing")
+
+    # audit
+    p_audit = sub.add_parser("audit", help="Document quality audit")
+    p_audit.add_argument("--kb", required=True)
+    p_audit.add_argument("--module", default=None)
+    p_audit.add_argument("--scope", default=None, help="Audit only this doc type (e.g. business-logic)")
+    p_audit.add_argument("--dry-run", action="store_true", help="Preview without applying fixes")
+    p_audit.add_argument("--force", action="store_true", help="Ignore progress, re-audit all")
+
+    # search
+    p_search = sub.add_parser("search", help="Search knowledge base")
+    p_search.add_argument("--kb", required=True)
+    p_search.add_argument("query")
+
+    # index
+    p_index = sub.add_parser("index", help="Build/rebuild index")
+    p_index.add_argument("--kb", required=True)
+
+    args = parser.parse_args()
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    setup_logging()
+
+    commands = {"init": cmd_init, "sync": cmd_sync, "audit": cmd_audit,
+                "search": cmd_search, "index": cmd_index}
+    commands[args.command](args)
+
+
+if __name__ == "__main__":
+    main()

engine/pipeline/__init__.py

@@ -0,0 +1,163 @@
+"""CLI pipeline framework — Pipeline class, PipelineBuilder, step registry.
+
+Orchestrates Step sequences with checkpoint validation, recovery, and rollback.
+
+Usage:
+    from engine.pipeline import Pipeline, PipelineBuilder, register_step
+
+    pipeline = PipelineBuilder("kb-init").add("fetch-source").add("extract-skeleton").build()
+    result = pipeline.execute(ctx)
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from core.interfaces import Step, StepResult, PipelineContext
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Step registry
+# ---------------------------------------------------------------------------
+
+_STEP_REGISTRY: dict[str, type[Step]] = {}
+
+
+def register_step(cls: type[Step]) -> type[Step]:
+    """Decorator to register a step class by its default name."""
+    # Use class attribute or class name as key
+    name = getattr(cls, "default_name", cls.__name__)
+    _STEP_REGISTRY[name] = cls
+    return cls
+
+
+def get_step(name: str) -> Step:
+    """Instantiate a registered step by name."""
+    if name not in _STEP_REGISTRY:
+        available = sorted(_STEP_REGISTRY.keys())
+        raise KeyError(f"Step '{name}' not registered. Available: {available}")
+    return _STEP_REGISTRY[name]()
+
+
+# ---------------------------------------------------------------------------
+# PipelineResult
+# ---------------------------------------------------------------------------
+
+@dataclass
+class PipelineResult:
+    """Outcome of a full pipeline execution."""
+    pipeline_name: str
+    step_results: list[tuple[str, StepResult]] = field(default_factory=list)
+    elapsed: float = 0.0
+
+    @property
+    def passed(self) -> bool:
+        return all(r.ok or r.skipped or r.delegated for _, r in self.step_results)
+
+    @property
+    def has_delegated(self) -> bool:
+        return any(r.delegated for _, r in self.step_results)
+
+    @property
+    def failed_steps(self) -> list[tuple[str, StepResult]]:
+        return [(n, r) for n, r in self.step_results if r.status == "failed"]
+
+    def summary(self) -> str:
+        ok = sum(1 for _, r in self.step_results if r.ok)
+        skip = sum(1 for _, r in self.step_results if r.skipped)
+        fail = sum(1 for _, r in self.step_results if r.status == "failed")
+        status = "PASS" if self.passed else "FAIL"
+        return f"{status}: {ok} ok, {skip} skipped, {fail} failed ({self.elapsed:.1f}s)"
+
+
+# ---------------------------------------------------------------------------
+# Pipeline
+# ---------------------------------------------------------------------------
+
+class Pipeline:
+    """Executes a sequence of Steps with error handling and rollback."""
+
+    def __init__(self, name: str, steps: list[Step]):
+        self.name = name
+        self.steps = steps
+
+    def execute(self, ctx: PipelineContext, checkpoint=None) -> PipelineResult:
+        result = PipelineResult(pipeline_name=self.name)
+        t0 = time.time()
+        executed: list[Step] = []
+        short_circuit = False
+
+        # Determine steps to skip if resuming from checkpoint
+        skip_steps: set[str] = set()
+        if checkpoint is not None:
+            from engine.pipeline.recovery import should_skip_step
+            skip_steps = {s.name for s in self.steps if should_skip_step(checkpoint, s.name)}
+
+        for step in self.steps:
+            if short_circuit:
+                result.step_results.append((step.name, StepResult(status="skipped", message="short-circuited")))
+                continue
+
+            if step.name in skip_steps:
+                result.step_results.append((step.name, StepResult(status="skipped", message="resumed (checkpoint)")))
+                continue
+
+            logger.info("[%s] running...", step.name)
+            try:
+                sr = step.run(ctx)
+            except Exception as e:
+                sr = StepResult(status="failed", message=str(e))
+
+            result.step_results.append((step.name, sr))
+            logger.info("[%s] %s", step.name, sr)
+
+            if sr.ok or sr.delegated:
+                executed.append(step)
+            elif sr.skipped:
+                if sr.details.get("short_circuit"):
+                    short_circuit = True
+            else:
+                # Failed — rollback
+                executed.append(step)
+                self._rollback(executed, ctx)
+                break
+
+        result.elapsed = time.time() - t0
+        return result
+
+    @staticmethod
+    def _rollback(executed: list[Step], ctx: PipelineContext) -> None:
+        for step in reversed(executed):
+            try:
+                step.rollback(ctx)
+            except Exception as e:
+                logger.error("[%s] rollback error: %s", step.name, e)
+
+    def describe(self) -> list[dict[str, str]]:
+        return [{"name": s.name, "checkpoint": s.checkpoint or ""} for s in self.steps]
+
+
+# ---------------------------------------------------------------------------
+# PipelineBuilder
+# ---------------------------------------------------------------------------
+
+class PipelineBuilder:
+    """Declarative pipeline composition from registered steps."""
+
+    def __init__(self, name: str):
+        self._name = name
+        self._steps: list[Step] = []
+
+    def add(self, step: Step | str) -> PipelineBuilder:
+        """Add a step (instance or registered name)."""
+        if isinstance(step, str):
+            step = get_step(step)
+        self._steps.append(step)
+        return self
+
+    def build(self) -> Pipeline:
+        return Pipeline(self._name, self._steps)