source-kb 0.2.2__py3-none-any.whl

engine/pipeline/steps/pregenerate.py

@@ -0,0 +1,130 @@
+"""Pre-generation steps — global metadata and dispatch plan.
+
+Steps that run before generate-docs to prepare shared context:
+- PregenerateMetadataStep: generates .meta/global-metadata.md
+- DispatchPlanStep: computes and writes .meta/dispatch-plan.md + dispatch-tasks.json
+
+These are required for prompt rendering (global_metadata variable) and
+for agent-mode task dispatching.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from core.interfaces import Step, StepResult, PipelineContext
+from engine.pipeline import register_step
+
+logger = logging.getLogger(__name__)
+
+
+@register_step
+class PregenerateMetadataStep(Step):
+    """Generate global-metadata.md for each module."""
+
+    default_name = "pregenerate-metadata"
+
+    def __init__(self):
+        super().__init__("pregenerate-metadata")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.skeleton.metadata import pregenerate
+
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        generated = []
+
+        for name in module_repos:
+            module_dir = ctx.knowledge_dir / name
+            try:
+                path = pregenerate(module_dir, module_name=name)
+                generated.append(name)
+                logger.info("[pregenerate-metadata] %s: %s", name, path)
+            except Exception as e:
+                logger.warning("[pregenerate-metadata] %s: %s", name, e)
+
+        if not generated:
+            return StepResult(status="skipped", message="No metadata generated")
+        return StepResult(status="ok", message=f"Generated metadata for {len(generated)} modules")
+
+
+@register_step
+class DispatchPlanStep(Step):
+    """Compute and write dispatch-plan.md + dispatch-tasks.json."""
+
+    default_name = "dispatch-plan"
+
+    def __init__(self):
+        super().__init__("dispatch-plan")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.preset import load_preset, get_doc_type_mapping
+        from core.skeleton.dispatch import compute_dispatch_plan
+        from core.skeleton.dispatch_render import render_markdown
+
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset = load_preset(preset_name)
+
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        plans_written = []
+
+        for name, repo_path in module_repos.items():
+            module_dir = ctx.knowledge_dir / name
+            source_cache = ctx.cache_dir / name
+
+            # Determine module type from config
+            module_type = _get_module_type(ctx, name)
+
+            try:
+                plan = compute_dispatch_plan(
+                    preset=preset,
+                    module_dir=module_dir,
+                    source_cache=source_cache,
+                    mode="output-only",
+                    module_name=name,
+                    module_type=module_type,
+                )
+            except Exception as e:
+                logger.warning("[dispatch-plan] %s: %s", name, e)
+                continue
+
+            # Write dispatch-plan.md
+            meta_dir = module_dir / ".meta"
+            meta_dir.mkdir(parents=True, exist_ok=True)
+
+            plan_md = render_markdown(plan, mode="output-only")
+            (meta_dir / "dispatch-plan.md").write_text(plan_md, encoding="utf-8")
+
+            # Write dispatch-tasks.json
+            from core.skeleton.dispatch_render import plan_to_tasks, write_shard_files
+            tasks = plan_to_tasks(
+                plan=plan, kb_name=ctx.kb_name, preset_name=preset_name,
+                preset=preset, knowledge_dir=ctx.knowledge_dir, mode="output-only",
+            )
+            (meta_dir / "dispatch-tasks.json").write_text(
+                json.dumps(tasks, ensure_ascii=False, indent=2), encoding="utf-8"
+            )
+
+            # Write shard file lists for split entries
+            write_shard_files(plan, module_dir)
+
+            plans_written.append(name)
+            logger.info("[dispatch-plan] %s: %d entries", name, len(plan.entries))
+
+        if not plans_written:
+            return StepResult(status="skipped", message="No dispatch plans generated")
+        return StepResult(
+            status="ok",
+            message=f"Dispatch plans for {len(plans_written)} modules",
+        )
+
+
+def _get_module_type(ctx: PipelineContext, module_name: str) -> str:
+    """Get module type from kb config."""
+    source = ctx.kb_config.get("source", {})
+    repos = source.get("repos", source.get("modules", []))
+    for repo in repos:
+        if repo.get("name") == module_name:
+            return repo.get("type", "service")
+    return "service"

engine/pipeline/steps/quality.py

@@ -0,0 +1,81 @@
+"""Quality validation steps — coverage, sampling, links, duplicates."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from core.interfaces import Step, StepResult, PipelineContext
+from engine.pipeline import register_step
+
+
+@register_step
+class ValidateStep(Step):
+    """Run all registered validators on generated documents."""
+
+    default_name = "validate"
+
+    def __init__(self):
+        super().__init__("validate", checkpoint="cp5")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.validators import run_all
+
+        modules = _get_module_dirs(ctx)
+        total_errors = 0
+        total_warnings = 0
+
+        for module_dir in modules:
+            result = run_all(module_dir)
+            total_errors += len(result.errors)
+            total_warnings += len(result.warnings)
+
+        status = "ok" if total_errors == 0 else "failed"
+        return StepResult(
+            status=status,
+            message=f"Validation: {total_errors} errors, {total_warnings} warnings",
+            details={"errors": total_errors, "warnings": total_warnings},
+        )
+
+
+@register_step
+class CoveragePreCheckStep(Step):
+    """Pre-check method coverage before generation."""
+
+    default_name = "coverage-precheck"
+
+    def __init__(self):
+        super().__init__("coverage-precheck")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        import json
+        from core.paths import resolve_skeleton
+        from core.skeleton.file_list import load_skeleton
+
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        total_methods = 0
+        high_methods = 0
+
+        for name in module_repos:
+            module_dir = ctx.knowledge_dir / name
+            entries = load_skeleton(module_dir)
+            for entry in entries:
+                methods = entry.get("methods", [])
+                total_methods += len(methods)
+                high_methods += sum(1 for m in methods if m.get("complexity") == "high")
+
+        ctx.state["coverage_precheck"] = {"total_methods": total_methods, "high_methods": high_methods}
+        return StepResult(
+            status="ok",
+            message=f"Pre-check: {total_methods} methods, {high_methods} high-complexity",
+        )
+
+
+def _get_module_dirs(ctx: PipelineContext) -> list[Path]:
+    """Get module directories to validate."""
+    if ctx.module:
+        d = ctx.knowledge_dir / ctx.module
+        return [d] if d.is_dir() else []
+    return sorted(
+        d for d in ctx.knowledge_dir.iterdir()
+        if d.is_dir() and not d.name.startswith(".")
+    )

engine/pipeline/steps/skeleton.py

@@ -0,0 +1,149 @@
+"""Skeleton extraction and file classification steps."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from core.interfaces import Step, StepResult, PipelineContext
+from core.paths import ensure_dir
+from engine.pipeline import register_step
+
+
+@register_step
+class ExtractSkeletonStep(Step):
+    """Extract skeleton from source repos using parser chain."""
+
+    default_name = "extract-skeleton"
+
+    def __init__(self):
+        super().__init__("extract-skeleton", checkpoint="cp3")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.skeleton.extract import extract_skeleton
+        from core.preset import load_preset
+
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset = load_preset(preset_name)
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        extracted = []
+
+        for name, repo_path in module_repos.items():
+            module_dir = ctx.knowledge_dir / name
+            module_dir.mkdir(parents=True, exist_ok=True)
+            ctx.state["current_module_dir"] = module_dir
+
+            branch = ctx.state.get("module_branches", {}).get(name, "main")
+            try:
+                entries = extract_skeleton(
+                    repo_path, preset, ref=f"origin/{branch}",
+                    output_dir=module_dir, split_by_package=True,
+                )
+                if entries:
+                    extracted.append(name)
+            except Exception as e:
+                print(f"  [{name}] skeleton failed: {e}", file=sys.stderr)
+
+        return StepResult(
+            status="ok" if extracted else "skipped",
+            message=f"Extracted skeletons for {len(extracted)} modules",
+            details={"extracted": extracted},
+        )
+
+
+@register_step
+class ExtractFileListStep(Step):
+    """Generate per-doc-type file lists from skeleton."""
+
+    default_name = "extract-file-list"
+
+    def __init__(self):
+        super().__init__("extract-file-list")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.skeleton.file_list import load_skeleton, extract_file_list
+        from core.preset import load_preset, get_doc_type_mapping
+        from core.paths import file_list_dir, file_list_path
+
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset = load_preset(preset_name)
+        doc_mapping = get_doc_type_mapping(preset)
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+
+        cache_dir = Path(ctx.kb_config["source"]["cache_dir"])
+        if not cache_dir.is_absolute():
+            cache_dir = (ctx.project_root / cache_dir).resolve()
+
+        total_lists = 0
+        for name in module_repos:
+            module_dir = ctx.knowledge_dir / name
+            source_cache = cache_dir / name
+            entries = load_skeleton(module_dir)
+            if not entries:
+                continue
+
+            fl_dir = file_list_dir(module_dir)
+            ensure_dir(fl_dir)
+
+            for doc_type_key, doc_filename in doc_mapping.items():
+                stem = doc_filename.removesuffix(".md")
+                files = extract_file_list(entries, preset, doc_type_key, str(source_cache))
+                output = file_list_path(module_dir, stem)
+                if files:
+                    output.write_text("\n".join(files) + "\n", encoding="utf-8")
+                    total_lists += 1
+                elif output.exists():
+                    output.unlink()
+
+        return StepResult(status="ok", message=f"Extracted {total_lists} file lists")
+
+
+@register_step
+class ClassifyStep(Step):
+    """LLM-assisted classification for uncovered files."""
+
+    default_name = "classify"
+
+    def __init__(self):
+        super().__init__("classify")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from core.skeleton.file_list import load_skeleton, check_coverage
+        from core.skeleton.classify import find_uncovered_files
+        from core.preset import load_preset
+        from core.paths import file_list_dir
+
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset = load_preset(preset_name)
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+
+        total_uncovered = 0
+        total_classified = 0
+
+        for name in module_repos:
+            module_dir = ctx.knowledge_dir / name
+            entries = load_skeleton(module_dir)
+            if not entries:
+                continue
+
+            fl_dir = file_list_dir(module_dir)
+            report = check_coverage(entries, preset, fl_dir if fl_dir.is_dir() else None)
+
+            if report.uncovered_count == 0:
+                continue
+
+            total_uncovered += report.uncovered_count
+
+            # LLM classification requires strategy — delegate to generate step
+            # For now, log uncovered files as warnings
+            for f in report.uncovered_files[:10]:
+                print(f"  [{name}] uncovered: {f}", file=sys.stderr)
+
+        if total_uncovered == 0:
+            return StepResult(status="ok", message="All files covered")
+
+        return StepResult(
+            status="ok",
+            message=f"{total_uncovered} uncovered files (LLM classification pending)",
+            details={"uncovered": total_uncovered, "classified": total_classified},
+        )

engine/pipeline/steps/source.py

@@ -0,0 +1,163 @@
+"""Source management steps — fetch, build, resolve dependencies."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+from core.interfaces import Step, StepResult, PipelineContext
+from core.git import ensure_repo
+from engine.pipeline import register_step
+
+
+@register_step
+class FetchSourceStep(Step):
+    """Clone/fetch source repositories."""
+
+    default_name = "fetch-source"
+
+    def __init__(self):
+        super().__init__("fetch-source", checkpoint="cp1")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        modules = _resolve_modules(ctx)
+        fetched, errors = [], []
+
+        for mod in modules:
+            name = mod["name"]
+            branch = mod.get("branch", "main")
+            url = mod.get("url")
+            local = mod.get("local")
+            try:
+                repo_path = ensure_repo(url, local, ctx.cache_dir, name, branch)
+                fetched.append(name)
+                ctx.state.setdefault("module_repos", {})[name] = repo_path
+                ctx.state.setdefault("module_branches", {})[name] = branch
+            except Exception as e:
+                errors.append(f"{name}: {e}")
+
+        if not fetched and errors:
+            return StepResult(status="failed", message="; ".join(errors))
+        msg = f"Fetched {len(fetched)} modules"
+        if errors:
+            msg += f" ({len(errors)} errors)"
+        return StepResult(status="ok", message=msg, details={"fetched": fetched})
+
+
+@register_step
+class BuildSourceStep(Step):
+    """Execute source build (Maven/Gradle) for jQAssistant support."""
+
+    default_name = "build-source"
+
+    def __init__(self):
+        super().__init__("build-source")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        import yaml
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset_path = ctx.project_root / "presets" / preset_name / "preset.yaml"
+        if not preset_path.exists():
+            return StepResult(status="skipped", message="No preset config")
+
+        with open(preset_path, encoding="utf-8") as f:
+            preset_config = yaml.safe_load(f)
+
+        build_cfg = preset_config.get("build", {})
+        if not build_cfg.get("enabled", False):
+            return StepResult(status="skipped", message="Build not enabled")
+
+        command = build_cfg.get("command", "mvn compile -DskipTests -q")
+        timeout = build_cfg.get("timeout", 300)
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        built, warnings = [], []
+
+        for name, repo_path in module_repos.items():
+            build_marker = build_cfg.get("marker_file", "pom.xml")
+            if not (repo_path / build_marker).exists():
+                continue
+            # Skip if already built
+            output_dir = build_cfg.get("output_dir", "target/classes")
+            if (repo_path / output_dir).is_dir():
+                built.append(f"{name} (cached)")
+                continue
+            try:
+                r = subprocess.run(command.split(), capture_output=True, encoding="utf-8",
+                                   timeout=timeout, cwd=str(repo_path))
+                if r.returncode == 0:
+                    built.append(name)
+                else:
+                    warnings.append(f"{name}: build failed (exit {r.returncode})")
+            except subprocess.TimeoutExpired:
+                warnings.append(f"{name}: build timed out")
+
+        return StepResult(status="ok", message=f"Built {len(built)} modules",
+                          details={"built": built, "warnings": warnings})
+
+
+@register_step
+class ResolveDepsStep(Step):
+    """Resolve Maven/Gradle dependencies (jar sources)."""
+
+    default_name = "resolve-deps"
+
+    def __init__(self):
+        super().__init__("resolve-deps", checkpoint="cp2")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        preset = ctx.kb_config.get("preset", "generic")
+        dep_script = ctx.project_root / "presets" / preset / "scripts" / "resolve_deps.py"
+        if not dep_script.exists():
+            return StepResult(status="skipped", message="No dependency resolver")
+
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        resolved, errors = [], []
+
+        for name, repo_path in module_repos.items():
+            try:
+                r = subprocess.run(
+                    [sys.executable, "-X", "utf8", str(dep_script), "--repo", str(repo_path),
+                     "--cache-dir", str(ctx.cache_dir)],
+                    capture_output=True, encoding="utf-8", timeout=300,
+                )
+                if r.returncode == 0:
+                    resolved.append(name)
+                else:
+                    errors.append(f"{name}: exit {r.returncode}")
+            except subprocess.TimeoutExpired:
+                errors.append(f"{name}: timed out")
+
+        msg = f"Resolved deps for {len(resolved)} modules"
+        if errors:
+            msg += f" ({len(errors)} warnings)"
+        return StepResult(status="ok", message=msg)
+
+
+def _resolve_modules(ctx: PipelineContext) -> list[dict]:
+    """Get module list from config, filtered by ctx.module and enabled flag."""
+    source = ctx.kb_config.get("source", {})
+    if source.get("structure") == "multi-repo":
+        modules = source.get("repos", [])
+    else:
+        modules = source.get("modules", [])
+
+    # Filter disabled modules
+    modules = [m for m in modules if m.get("enabled", True) is not False]
+
+    # Filter excluded modules
+    excluded = _parse_excluded_modules(ctx.kb_config)
+    if excluded:
+        modules = [m for m in modules if m.get("name") not in excluded]
+
+    if ctx.module:
+        modules = [m for m in modules if m.get("name") == ctx.module]
+    return modules
+
+
+def _parse_excluded_modules(kb_config: dict) -> set[str]:
+    """Parse excluded_modules from kb config."""
+    excluded = kb_config.get("excluded_modules", [])
+    if isinstance(excluded, str):
+        excluded = [s.strip() for s in excluded.split(",") if s.strip()]
+    return set(excluded)

engine/pipeline/steps/sync.py

@@ -0,0 +1,117 @@
+"""Sync pipeline steps — cache update and change detection.
+
+Steps:
+- UpdateCacheStep: git pull on source caches
+- DetectChangesStep: detect changed files via git diff + baseline
+
+Requirements: Req 11, 15, 26
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from core.interfaces import Step, StepResult, PipelineContext
+from core.git import fetch_with_retry, get_head_commit, diff_files
+from engine.pipeline import register_step
+
+logger = logging.getLogger(__name__)
+
+
+@register_step
+class UpdateCacheStep(Step):
+    """Git pull on source cache directories."""
+
+    default_name = "update-cache"
+
+    def __init__(self):
+        super().__init__("update-cache")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from engine.pipeline.steps.source import _resolve_modules
+
+        modules = _resolve_modules(ctx)
+        updated = []
+
+        for mod in modules:
+            name = mod["name"]
+            repo_dir = ctx.cache_dir / name
+            if not (repo_dir / ".git").exists():
+                continue
+            branch = mod.get("branch", "main")
+            try:
+                fetch_with_retry(repo_dir, branch)
+                updated.append(name)
+                ctx.state.setdefault("module_repos", {})[name] = repo_dir
+                ctx.state.setdefault("module_branches", {})[name] = branch
+            except Exception as e:
+                logger.warning("[update-cache] %s: %s", name, e)
+
+        return StepResult(status="ok", message=f"Updated {len(updated)} caches")
+
+
+@register_step
+class DetectChangesStep(Step):
+    """Detect changed files via git diff against stored baseline."""
+
+    default_name = "detect-changes"
+
+    def __init__(self):
+        super().__init__("detect-changes")
+
+    def run(self, ctx: PipelineContext) -> StepResult:
+        from engine.pipeline.steps.sync_finalize import _load_baseline
+        from core.preset import load_preset
+
+        module_repos: dict[str, Path] = ctx.state.get("module_repos", {})
+        if not module_repos:
+            return StepResult(status="skipped", message="No repos",
+                              details={"short_circuit": True})
+
+        preset_name = ctx.kb_config.get("preset", "generic")
+        preset = load_preset(preset_name)
+        limits = preset.get("limits", {})
+        max_changed = limits.get("sync_max_changed_files", 20)
+
+        baseline = _load_baseline(ctx)
+        all_changed: list[str] = []
+        change_details: dict[str, list[dict]] = {}
+
+        for name, repo_path in module_repos.items():
+            branch = ctx.state.get("module_branches", {}).get(name, "main")
+            new_commit = get_head_commit(repo_path, ref=f"origin/{branch}")
+            if not new_commit:
+                continue
+
+            old_commit = baseline.get(name)
+            if not old_commit or old_commit == new_commit:
+                continue
+
+            try:
+                changes = diff_files(repo_path, old_commit, new_commit)
+            except Exception as e:
+                logger.error("[detect-changes] %s diff failed: %s", name, e)
+                continue
+
+            files = [c["file"] for c in changes]
+            all_changed.extend(files)
+            change_details[name] = changes
+
+        if not all_changed:
+            return StepResult(status="skipped", message="No changes detected",
+                              details={"short_circuit": True})
+
+        if len(all_changed) > max_changed:
+            return StepResult(
+                status="failed",
+                message=f"Too many changes ({len(all_changed)} files) — use kb-audit",
+                details={"file_count": len(all_changed), "suggest_audit": True},
+            )
+
+        ctx.state["changed_files"] = all_changed
+        ctx.state["change_details"] = change_details
+        return StepResult(
+            status="ok",
+            message=f"Detected {len(all_changed)} changed files in {len(change_details)} modules",
+        )