source-kb 0.2.2__py3-none-any.whl

engine/pipeline/steps/sync_finalize.py

@@ -0,0 +1,237 @@
+"""Sync finalize steps — post-update validation, baseline, and repair.
+
+Steps:
+- SharedDocsCascadeStep: update _shared/ docs when cross-module info changes
+- UpdateBaselineStep: update baseline ONLY after validation passes
+- AnchorFixStep: repair broken cross-document links
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from core.interfaces import Step, StepResult, PipelineContext
+from core.git import get_head_commit
+from engine.pipeline import register_step
+
+logger = logging.getLogger(__name__)
+
+
+@register_step
+class SharedDocsCascadeStep(Step):
+    """Update _shared/ documents when module changes affect cross-module info.
+
+    Triggers:
+    - New/modified @FeignClient -> _shared/cross-module-calls.md
+    - New/modified error codes -> _shared/error-codes.md
+    - New/modified @TableName -> _shared/database-ddl.md
+    """
+
+    default_name = "shared-docs-cascade"
+
+    def __init__(self):
+        super().__init__("shared-docs-cascade")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        affected_docs = ctx.state.get("affected_docs", {})
+        if not affected_docs:
+            return StepResult(status="skipped", message="No affected docs")
+
+        shared_dir = ctx.knowledge_dir / "_shared"
+        if not shared_dir.is_dir():
+            return StepResult(status="skipped", message="No _shared/ directory")
+
+        triggers = _detect_shared_doc_triggers(affected_docs)
+        if not triggers:
+            return StepResult(status="skipped", message="No shared doc impact")
+
+        logger.info("[shared-docs-cascade] Detected triggers: %s", triggers)
+        return StepResult(
+            status="ok",
+            message=f"Detected {len(triggers)} shared doc triggers (update deferred)",
+            details={"triggers": triggers},
+        )
+
+
+@register_step
+class UpdateBaselineStep(Step):
+    """Update sync baseline ONLY after all validations pass.
+
+    Preconditions:
+    - Not too many failures (fail ratio < 50%)
+    - No fatal errors in prior steps
+    """
+
+    default_name = "update-baseline"
+
+    def __init__(self):
+        super().__init__("update-baseline")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        failed_sections = ctx.state.get("failed_sections", [])
+        updated_sections = ctx.state.get("updated_sections", [])
+        total = len(updated_sections) + len(failed_sections)
+
+        if total > 0 and len(failed_sections) > 0:
+            fail_ratio = len(failed_sections) / total
+            if fail_ratio > 0.5:
+                return StepResult(
+                    status="skipped",
+                    message=f"Too many failures ({len(failed_sections)}/{total}), baseline not updated",
+                )
+
+        if ctx.state.get("dry_run", False):
+            _write_dry_run_report(ctx)
+            return StepResult(status="skipped", message="Dry-run: baseline not updated")
+
+        _update_baseline(ctx)
+        return StepResult(status="ok", message="Baseline updated")
+
+
+@register_step
+class AnchorFixStep(Step):
+    """Repair broken cross-document anchor links after document updates."""
+
+    default_name = "anchor-fix"
+
+    def __init__(self):
+        super().__init__("anchor-fix")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.skeleton.anchor_fix import fix_anchors
+
+        updated_sections = ctx.state.get("updated_sections", [])
+        if not updated_sections:
+            return StepResult(status="skipped", message="No updates to check")
+
+        module_names = {s["module"] for s in updated_sections}
+        total_fixed = 0
+        total_degraded = 0
+
+        for module_name in module_names:
+            module_dir = ctx.knowledge_dir / module_name
+            if not module_dir.is_dir():
+                continue
+
+            result = fix_anchors(module_dir)
+            total_fixed += result.links_fixed
+            total_degraded += result.links_degraded
+
+        if total_fixed or total_degraded:
+            return StepResult(
+                status="ok",
+                message=f"Anchor fix: {total_fixed} fixed, {total_degraded} degraded",
+            )
+        return StepResult(status="ok", message="All links valid")
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _load_baseline(ctx: PipelineContext) -> dict[str, str]:
+    """Load sync baseline (module -> commit hash)."""
+    state_dir = ctx.knowledge_dir.parent / ".source-kb"
+    state_file = state_dir / "sync-state.json"
+    if not state_file.exists():
+        return {}
+    try:
+        all_state = json.loads(state_file.read_text(encoding="utf-8"))
+        return all_state.get(ctx.kb_name, {})
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def _update_baseline(ctx: PipelineContext) -> None:
+    """Update sync baseline with current HEAD commits."""
+    module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+    if not module_repos:
+        return
+
+    state_dir = ctx.knowledge_dir.parent / ".source-kb"
+    state_dir.mkdir(parents=True, exist_ok=True)
+    state_file = state_dir / "sync-state.json"
+
+    all_state: dict = {}
+    if state_file.exists():
+        try:
+            all_state = json.loads(state_file.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    kb_state = all_state.setdefault(ctx.kb_name, {})
+    for name, repo_path in module_repos.items():
+        branch = ctx.state.get("module_branches", {}).get(name, "main")
+        commit = get_head_commit(repo_path, ref=f"origin/{branch}")
+        if commit:
+            kb_state[name] = commit
+
+    state_file.write_text(
+        json.dumps(all_state, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+
+
+def _classify_changes(changed_files: list[str], preset: dict) -> dict[str, list[str]]:
+    """Classify changed files into affected doc types."""
+    from core.preset_classify import classify_file
+
+    classification: dict[str, list[str]] = {}
+    for fpath in changed_files:
+        categories = classify_file(preset, fpath)
+        if categories:
+            for cat in categories:
+                classification.setdefault(cat, []).append(fpath)
+        else:
+            fallback_cat = "source"
+            doc_types = preset.get("doc_types", {})
+            for dt_key, dt_config in doc_types.items():
+                if isinstance(dt_config, dict) and not dt_config.get("conditional", True):
+                    fallback_cat = dt_key
+                    break
+            classification.setdefault(fallback_cat, []).append(fpath)
+
+    return classification
+
+
+def _write_dry_run_report(ctx: PipelineContext) -> None:
+    """Write dry-run report with proposed changes."""
+    changes = ctx.state.get("dry_run_changes", [])
+    updated = ctx.state.get("updated_sections", [])
+
+    report = {
+        "operation": "kb-sync",
+        "mode": "dry-run",
+        "proposed_changes": changes + [
+            {"doc_path": s["doc"], "section": s["heading"],
+             "action": s["action"], "module": s["module"]}
+            for s in updated
+        ],
+        "summary": {
+            "total_changes": len(changes) + len(updated),
+        },
+    }
+
+    report_path = ctx.knowledge_dir / ".dry-run-report.json"
+    report_path.write_text(
+        json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    logger.info("[dry-run] Report written: %s", report_path)
+
+
+def _detect_shared_doc_triggers(affected_docs: dict[str, list[str]]) -> list[str]:
+    """Detect if changes affect shared documents."""
+    triggers: list[str] = []
+
+    if any(cat in affected_docs for cat in {"feign", "feign-external"}):
+        triggers.append("cross-module-calls")
+
+    if "error-handling" in affected_docs:
+        triggers.append("error-codes")
+
+    if "model" in affected_docs:
+        triggers.append("database-ddl")
+
+    return triggers

engine/pipeline/steps/sync_update.py

@@ -0,0 +1,341 @@
+"""Sync update step — LLM-powered incremental document section updates.
+
+Steps:
+- IncrementalUpdateStep: LLM call per affected section, or delegated manifest
+
+Requirements: Req 11, 15, 19, 24
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, TYPE_CHECKING
+
+from core.interfaces import Step, StepResult, PipelineContext, LlmRequest
+from core.utils import validate_path_within_bounds
+from engine.pipeline import register_step
+
+if TYPE_CHECKING:
+    from core.skeleton.impact import SectionImpact
+
+logger = logging.getLogger(__name__)
+
+
+@register_step
+class IncrementalUpdateStep(Step):
+    """LLM-powered incremental document section updates.
+
+    Modes:
+    - CLI (api): Impact analysis -> LLM call -> validate -> write
+    - Delegated: Generate .sync-manifest.json for Agent dispatch
+    """
+
+    default_name = "incremental-update"
+
+    def __init__(self):
+        super().__init__("incremental-update", checkpoint="incremental-update")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.preset import load_preset
+        from core.skeleton.impact import analyze_impact
+        from core.docs.section_updater import replace_section, append_section
+        from core.prompt.response_parser import validate_sync_response
+        from engine.strategies import create_strategy
+        from engine.pipeline.steps.generate import _make_config_obj
+        from engine.pipeline.steps.sync_finalize import _classify_changes
+
+        changed_files = ctx.state.get("changed_files", [])
+        if not changed_files:
+            return StepResult(status="skipped", message="No changes to process")
+
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset = load_preset(preset_name)
+        config_obj = _make_config_obj(ctx)
+
+        change_details: dict[str, list[dict]] = ctx.state.get("change_details", {})
+
+        # Create strategy — delegated mode is handled via response.status, not upfront check
+        strategy = create_strategy(config_obj)
+        dry_run = ctx.state.get("dry_run", False)
+
+        total_updated = 0
+        total_failed = 0
+        all_updated_sections: list[dict] = []
+        all_failed_sections: list[dict] = []
+
+        for module_name, changes in change_details.items():
+            module_dir = ctx.knowledge_dir / module_name
+            source_cache = ctx.cache_dir / module_name
+
+            if not module_dir.is_dir():
+                continue
+
+            module_files = [c["file"] for c in changes]
+
+            plan = analyze_impact(
+                changed_files=module_files,
+                module_dir=module_dir,
+                preset=preset,
+                source_cache=source_cache,
+                change_details=changes,
+            )
+
+            if not plan.sections:
+                logger.info("[sync] %s: no sections affected", module_name)
+                continue
+
+            for impact in plan.sections:
+                doc_path = module_dir / impact.doc_path
+                if not validate_path_within_bounds(doc_path, module_dir):
+                    logger.warning("[sync] Path traversal blocked: %s", doc_path)
+                    total_failed += 1
+                    continue
+                if not doc_path.exists() and impact.change_type != "create":
+                    all_failed_sections.append({
+                        "module": module_name, "doc": impact.doc_path,
+                        "heading": impact.section_heading, "reason": "doc not found",
+                    })
+                    total_failed += 1
+                    continue
+
+                old_content = ""
+                if doc_path.exists() and impact.change_type == "update":
+                    old_content = _read_section_content(doc_path, impact.section_heading)
+
+                system_prompt = _build_sync_system_prompt(impact.doc_path)
+                user_prompt = _build_sync_user_prompt(
+                    impact, old_content, module_files, source_cache
+                )
+
+                if dry_run:
+                    ctx.state.setdefault("dry_run_changes", []).append({
+                        "module": module_name,
+                        "doc_path": impact.doc_path,
+                        "section": impact.section_heading,
+                        "action": impact.change_type,
+                        "affected_files": impact.affected_files,
+                    })
+                    total_updated += 1
+                    continue
+
+                request = LlmRequest(system=system_prompt, user=user_prompt)
+                try:
+                    response = strategy.call(request)
+                except Exception as e:
+                    logger.error("[sync] LLM call failed for %s/%s: %s",
+                                module_name, impact.section_heading, e)
+                    all_failed_sections.append({
+                        "module": module_name, "doc": impact.doc_path,
+                        "heading": impact.section_heading, "reason": str(e),
+                    })
+                    total_failed += 1
+                    continue
+
+                if response.status == "delegated":
+                    # Accumulate for manifest generation
+                    ctx.state.setdefault("_delegated_sections", []).append({
+                        "module": module_name,
+                        "doc_path": impact.doc_path,
+                        "section_heading": impact.section_heading,
+                        "action": impact.change_type,
+                        "affected_files": impact.affected_files,
+                        "confidence": impact.confidence,
+                    })
+                    total_updated += 1
+                    continue
+
+                if response.status != "done":
+                    all_failed_sections.append({
+                        "module": module_name, "doc": impact.doc_path,
+                        "heading": impact.section_heading,
+                        "reason": f"LLM status: {response.status}",
+                    })
+                    total_failed += 1
+                    continue
+
+                is_valid, content = validate_sync_response(response.content, old_content)
+                if not is_valid:
+                    logger.warning("[sync] Invalid response for %s/%s: %s",
+                                  module_name, impact.section_heading, content)
+                    _mark_unverified(doc_path, impact.section_heading)
+                    all_failed_sections.append({
+                        "module": module_name, "doc": impact.doc_path,
+                        "heading": impact.section_heading, "reason": content,
+                    })
+                    total_failed += 1
+                    continue
+
+                if impact.change_type == "update":
+                    replace_section(doc_path, impact.section_heading, content)
+                elif impact.change_type == "create":
+                    append_section(doc_path, impact.section_heading, content)
+
+                all_updated_sections.append({
+                    "module": module_name, "doc": impact.doc_path,
+                    "heading": impact.section_heading, "action": impact.change_type,
+                })
+                total_updated += 1
+
+        ctx.state["updated_sections"] = all_updated_sections
+        ctx.state["failed_sections"] = all_failed_sections
+
+        affected_docs = _classify_changes(changed_files, preset)
+        ctx.state["affected_docs"] = affected_docs
+
+        # If delegated mode accumulated sections, write manifest and return delegated
+        delegated_sections = ctx.state.pop("_delegated_sections", [])
+        if delegated_sections:
+            return self._write_manifest(ctx, delegated_sections, change_details)
+
+        if total_updated == 0 and total_failed == 0:
+            return StepResult(status="skipped", message="No sections affected")
+
+        return StepResult(
+            status="ok",
+            message=f"Updated {total_updated} sections, {total_failed} failed",
+            details={"updated": total_updated, "failed": total_failed},
+        )
+
+    def _write_manifest(
+        self, ctx: PipelineContext, delegated_sections: list[dict],
+        change_details: dict[str, list[dict]],
+    ) -> StepResult:
+        """Write sync manifest from accumulated delegated responses."""
+        manifest = {"operation": "kb-sync", "modules": []}
+
+        # Group sections by module
+        by_module: dict[str, list[dict]] = {}
+        for s in delegated_sections:
+            by_module.setdefault(s["module"], []).append(s)
+
+        for module_name, sections in by_module.items():
+            module_files = [c["file"] for c in change_details.get(module_name, [])]
+            module_entry = {
+                "name": module_name,
+                "changed_files": module_files,
+                "affected_sections": [
+                    {
+                        "doc_path": s["doc_path"],
+                        "section_heading": s["section_heading"],
+                        "action": s["action"],
+                        "affected_files": s["affected_files"],
+                        "confidence": s.get("confidence", 1.0),
+                    }
+                    for s in sections
+                ],
+            }
+            manifest["modules"].append(module_entry)
+
+        manifest_path = ctx.knowledge_dir / ".sync-manifest.json"
+        manifest_path.write_text(
+            json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
+        )
+
+        total_sections = sum(len(m["affected_sections"]) for m in manifest["modules"])
+        return StepResult(
+            status="delegated",
+            message=f"Manifest: {total_sections} sections in {len(manifest['modules'])} modules",
+            details={"manifest_path": str(manifest_path), "sections": total_sections},
+        )
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _read_section_content(doc_path: Path, heading: str) -> str:
+    """Read the content of a specific section from a document."""
+    from core.docs.section_updater import find_section_boundaries
+
+    try:
+        content = doc_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return ""
+
+    lines = content.split("\n")
+    if lines and lines[-1] == "":
+        lines = lines[:-1]
+
+    bounds = find_section_boundaries(lines, heading)
+    if bounds is None:
+        return ""
+
+    start, end = bounds
+    section_lines = lines[start + 1:end]
+    return "\n".join(section_lines).strip()
+
+
+def _mark_unverified(doc_path: Path, heading: str) -> None:
+    """Mark a section as unverified by prepending a warning marker."""
+    from core.docs.section_updater import find_section_boundaries
+
+    try:
+        content = doc_path.read_text(encoding="utf-8")
+    except OSError:
+        return
+
+    lines = content.split("\n")
+    if lines and lines[-1] == "":
+        lines = lines[:-1]
+
+    bounds = find_section_boundaries(lines, heading)
+    if bounds is None:
+        return
+
+    start, end = bounds
+    for i in range(start + 1, end):
+        if lines[i].strip():
+            if not lines[i].startswith("[UNVERIFIED]"):
+                lines[i] = f"[UNVERIFIED] {lines[i]}"
+            break
+
+    doc_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _build_sync_system_prompt(doc_type: str) -> str:
+    """Build system prompt for sync section update."""
+    from core.preset import find_preset_template
+
+    template_path = find_preset_template("java-spring", "sync-system.md")
+    if template_path and template_path.exists():
+        try:
+            return template_path.read_text(encoding="utf-8").strip()
+        except OSError:
+            pass
+    return (
+        "You are a knowledge base maintainer. Update the specific document section "
+        "based on source code changes.\n\n"
+        "Rules:\n"
+        "- Only rewrite the affected section content, maintain the style of the rest\n"
+        "- Describe from a business perspective, not code translation\n"
+        "- Output section body only (without the heading line)\n"
+        "- Do not fabricate; mark uncertain content with [UNVERIFIED]"
+    )
+
+
+def _build_sync_user_prompt(
+    impact: SectionImpact, old_content: str, changed_files: list[str], source_cache: Path
+) -> str:
+    """Build user prompt for sync section update."""
+    parts = [
+        f"Document: {impact.doc_path}",
+        f"Section: {impact.section_heading}",
+        f"Change type: {impact.change_type}",
+        "",
+        "Current section content:",
+        old_content if old_content else "(empty)",
+        "",
+        "Changed files:",
+    ]
+    for f in impact.affected_files[:10]:
+        parts.append(f"  - {f}")
+    if len(impact.affected_files) > 10:
+        parts.append(f"  ... and {len(impact.affected_files) - 10} more files")
+
+    parts.append("")
+    parts.append("Rewrite this section based on the source code changes.")
+
+    return "\n".join(parts)

engine/pipelines.py

@@ -0,0 +1,91 @@
+"""Pipeline builders — compose steps into named pipelines.
+
+Usage:
+    from engine.pipelines import build_init_pipeline, build_sync_pipeline, build_audit_pipeline
+
+    pipeline = build_init_pipeline()
+    result = pipeline.execute(ctx)
+"""
+
+from __future__ import annotations
+
+from engine.pipeline import Pipeline
+from engine.pipeline.steps.lock import AcquireLockStep, ReleaseLockStep
+from engine.pipeline.steps.source import FetchSourceStep, BuildSourceStep, ResolveDepsStep
+from engine.pipeline.steps.skeleton import ExtractSkeletonStep, ExtractFileListStep, ClassifyStep
+from engine.pipeline.steps.pregenerate import PregenerateMetadataStep, DispatchPlanStep
+from engine.pipeline.steps.preflight import PreflightCheckStep
+from engine.pipeline.steps.generate import GenerateDocsStep
+from engine.pipeline.steps.quality import ValidateStep, CoveragePreCheckStep
+from engine.pipeline.steps.finalize import (
+    MergeShardsStep, DeduplicateStep, RebuildIndexStep,
+    CleanProgressStep, SharedDocsStep,
+)
+from engine.pipeline.steps.sync import (
+    UpdateCacheStep, DetectChangesStep,
+)
+from engine.pipeline.steps.sync_update import IncrementalUpdateStep
+from engine.pipeline.steps.sync_finalize import (
+    SharedDocsCascadeStep, UpdateBaselineStep, AnchorFixStep,
+)
+from engine.pipeline.steps.audit import FetchAndScaleStep, AuditDocsStep
+from engine.pipeline.steps.audit_apply import ApplyFixesStep, GenerateReportStep
+
+
+def build_init_pipeline() -> Pipeline:
+    """Full kb-init pipeline."""
+    return Pipeline("kb-init", [
+        AcquireLockStep(timeout_minutes=120),
+        FetchSourceStep(),
+        BuildSourceStep(),
+        ResolveDepsStep(),
+        ExtractSkeletonStep(),
+        ExtractFileListStep(),
+        ClassifyStep(),
+        CoveragePreCheckStep(),
+        PregenerateMetadataStep(),
+        DispatchPlanStep(),
+        PreflightCheckStep(),
+        GenerateDocsStep(),
+        MergeShardsStep(),
+        DeduplicateStep(),
+        ValidateStep(),
+        SharedDocsStep(),
+        CleanProgressStep(),
+        RebuildIndexStep(),
+        ReleaseLockStep(),
+    ])
+
+
+def build_sync_pipeline() -> Pipeline:
+    """Incremental kb-sync pipeline."""
+    return Pipeline("kb-sync", [
+        AcquireLockStep(timeout_minutes=30),
+        UpdateCacheStep(),
+        BuildSourceStep(),
+        DetectChangesStep(),
+        ExtractSkeletonStep(),
+        ExtractFileListStep(),
+        IncrementalUpdateStep(),
+        SharedDocsCascadeStep(),
+        ValidateStep(),
+        UpdateBaselineStep(),
+        AnchorFixStep(),
+        RebuildIndexStep(),
+        ReleaseLockStep(),
+    ])
+
+
+def build_audit_pipeline() -> Pipeline:
+    """kb-audit pipeline."""
+    return Pipeline("kb-audit", [
+        AcquireLockStep(timeout_minutes=30),
+        FetchAndScaleStep(),
+        AuditDocsStep(),
+        ApplyFixesStep(),
+        AnchorFixStep(),
+        ValidateStep(),
+        RebuildIndexStep(),
+        GenerateReportStep(),
+        ReleaseLockStep(),
+    ])