devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/source_doc_mutation_eval.py

@@ -0,0 +1,417 @@
+from __future__ import annotations
+
+import json
+import shutil
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from .agentic_prompts import load_agentic_prompt_lines
+from .agentic_runtime import run_agent_step
+from .source_doc_mutation_dag import PROJECT_DOC_TARGETS, build_source_doc_mutation_request, run_source_doc_mutation_dag
+
+CANONICAL_SOURCE_DOCS = (
+    "product_brief.json",
+    "user_workflows.json",
+    "domain_entities.json",
+    "assumptions_registry.json",
+)
+
+EXCLUDED_PROJECT_DOC_MATRIX_DOCS = {"index.md"}
+
+
+class DocumentAssessment(BaseModel):
+    doc_name: str
+    doc_family: str
+    quality: float
+    completeness: float
+    notes: list[str] = []
+
+
+class MutationDocumentAssessment(DocumentAssessment):
+    appropriate_update: float
+    thoroughness: float
+    accuracy: float
+
+
+@dataclass(frozen=True)
+class SourceDocMutationEvalResult:
+    run_id: str
+    started_at: str
+    eval_root: Path
+    payload: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class TargetedMutationBaseline:
+    eval_root: Path
+    source_docs_root: Path
+    project_docs_root: Path
+    manifest: dict[str, Any]
+
+
+def _now_run_id() -> tuple[str, str]:
+    started_at = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+    run_id = started_at.replace("-", "").replace(":", "")
+    return run_id, started_at
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _copy_tree(src: Path, dst: Path) -> list[str]:
+    copied: list[str] = []
+    dst.mkdir(parents=True, exist_ok=True)
+    for path in sorted(src.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(src)
+        out = dst / rel
+        out.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(path, out)
+        copied.append(rel.as_posix())
+    return copied
+
+
+def _mean(values: list[float]) -> float:
+    if not values:
+        return 0.0
+    return round(sum(values) / len(values), 6)
+
+
+def _existing_source_refs() -> list[str]:
+    return [f"ai_docs/context/source_docs/{name}" for name in CANONICAL_SOURCE_DOCS]
+
+
+def _doc_paths(eval_root: Path) -> tuple[Path, Path]:
+    return eval_root / "ai_docs" / "context" / "source_docs", eval_root / "ai_docs" / "context" / "project_docs"
+
+
+def _baseline_doc_paths(eval_root: Path) -> tuple[Path, Path]:
+    return eval_root / "baseline" / "source_docs", eval_root / "baseline" / "project_docs"
+
+
+def _assess_doc(*, repo_root: Path, stage_name: str, output_model: type[BaseModel], context_payload: dict[str, Any], guidance: list[str]) -> BaseModel:
+    model, _ = run_agent_step(
+        repo_root=repo_root,
+        stage_name=stage_name,
+        output_model=output_model,
+        context_payload=context_payload,
+        guidance=guidance,
+    )
+    return model
+
+
+def _resolve_targeted_mutation_baseline(*, evals_root: Path, baseline_eval_root: Path | None = None) -> TargetedMutationBaseline:
+    if baseline_eval_root is not None:
+        resolved = baseline_eval_root.resolve()
+        manifest_path = resolved / "eval_manifest.json"
+        if manifest_path.exists():
+            candidates = [resolved]
+        elif resolved.exists() and resolved.is_dir():
+            candidates = sorted((path.resolve() for path in resolved.iterdir() if path.is_dir()), reverse=True)
+        else:
+            candidates = [resolved]
+    else:
+        candidates = sorted((path.resolve() for path in evals_root.iterdir() if path.is_dir()), reverse=True)
+    for candidate in candidates:
+        manifest_path = candidate / "eval_manifest.json"
+        if not manifest_path.exists():
+            continue
+        manifest = _load_json(manifest_path)
+        if manifest.get("eval_type") != "sparse_input":
+            continue
+        source_docs_root = candidate / "ai_docs" / "context" / "source_docs"
+        project_docs_root = candidate / "ai_docs" / "context" / "project_docs"
+        if not source_docs_root.exists() or not project_docs_root.exists():
+            continue
+        return TargetedMutationBaseline(
+            eval_root=candidate,
+            source_docs_root=source_docs_root,
+            project_docs_root=project_docs_root,
+            manifest=manifest,
+        )
+    if baseline_eval_root is not None:
+        raise FileNotFoundError(f"Targeted mutation baseline not found or invalid at {baseline_eval_root}")
+    raise FileNotFoundError(
+        f"No sparse_input eval baseline found under {evals_root}. Run a sparse_input eval first or pass baseline_eval_root explicitly."
+    )
+
+
+def _summarize_assessments(eval_root: Path, run_id: str, started_at: str, assessments: list[DocumentAssessment]) -> dict[str, Any]:
+    source_docs = [a for a in assessments if a.doc_family == "source_docs"]
+    project_docs = [a for a in assessments if a.doc_family == "project_docs" and a.doc_name not in EXCLUDED_PROJECT_DOC_MATRIX_DOCS]
+    excluded_project_docs = [a for a in assessments if a.doc_family == "project_docs" and a.doc_name in EXCLUDED_PROJECT_DOC_MATRIX_DOCS]
+    overall_assessments = source_docs + project_docs
+    source_index = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "doc_family": "source_docs",
+        "docs": [a.model_dump() for a in source_docs],
+        "mean_quality": _mean([a.quality for a in source_docs]),
+        "mean_completeness": _mean([a.completeness for a in source_docs]),
+    }
+    project_index = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "doc_family": "project_docs",
+        "docs": [a.model_dump() for a in project_docs],
+        "excluded_docs": [a.doc_name for a in excluded_project_docs],
+        "mean_quality": _mean([a.quality for a in project_docs]),
+        "mean_completeness": _mean([a.completeness for a in project_docs]),
+    }
+    overall = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "source_docs": {
+            "quality": source_index["mean_quality"],
+            "completeness": source_index["mean_completeness"],
+        },
+        "project_docs": {
+            "quality": project_index["mean_quality"],
+            "completeness": project_index["mean_completeness"],
+        },
+        "overall": {
+            "quality": _mean([a.quality for a in overall_assessments]),
+            "completeness": _mean([a.completeness for a in overall_assessments]),
+        },
+    }
+    _write_json(eval_root / "indexes" / "source_docs_index.json", source_index)
+    _write_json(eval_root / "indexes" / "project_docs_index.json", project_index)
+    _write_json(eval_root / "metrics" / "quality.json", {
+        "run_id": run_id,
+        "started_at": started_at,
+        "per_document": [{"doc_name": a.doc_name, "doc_family": a.doc_family, "quality": a.quality} for a in assessments],
+        "source_docs": source_index["mean_quality"],
+        "project_docs": project_index["mean_quality"],
+        "excluded_project_docs": sorted(EXCLUDED_PROJECT_DOC_MATRIX_DOCS),
+        "overall": overall["overall"]["quality"],
+    })
+    _write_json(eval_root / "metrics" / "completeness.json", {
+        "run_id": run_id,
+        "started_at": started_at,
+        "per_document": [{"doc_name": a.doc_name, "doc_family": a.doc_family, "completeness": a.completeness} for a in assessments],
+        "source_docs": source_index["mean_completeness"],
+        "project_docs": project_index["mean_completeness"],
+        "excluded_project_docs": sorted(EXCLUDED_PROJECT_DOC_MATRIX_DOCS),
+        "overall": overall["overall"]["completeness"],
+    })
+    return overall
+
+
+def run_sparse_input_eval(*, repo_root: Path, text: str, agent_runtime_mode: str = "real", project_id: str = "eval_sparse_input", idea_id: str = "eval_sparse_input_001", pipeline_key: str = "eval_sparse_input", evals_root: Path | None = None) -> SourceDocMutationEvalResult:
+    run_id, started_at = _now_run_id()
+    request = build_source_doc_mutation_request(
+        project_id=project_id,
+        idea_id=idea_id,
+        repo_root=repo_root,
+        raw_text=text,
+        grounded_raw_text=text,
+        mode="ideation_mutation",
+        existing_source_doc_refs=_existing_source_refs(),
+        pipeline_key=pipeline_key,
+        agent_runtime_mode=agent_runtime_mode,
+    )
+    dag_result = run_source_doc_mutation_dag(request=request)
+    eval_root = (evals_root or (repo_root / "evals")) / run_id
+    source_dst, project_dst = _doc_paths(eval_root)
+    source_copied = _copy_tree(repo_root / "ai_docs" / "context" / "source_docs", source_dst)
+    project_copied = _copy_tree(repo_root / "ai_docs" / "context" / "v2" / "project_docs" / "source_docs", project_dst)
+
+    assessments: list[DocumentAssessment] = []
+    for family, root in (("source_docs", source_dst), ("project_docs", project_dst)):
+        for path in sorted(root.glob("*.json" if family == "source_docs" else "*.md")):
+            if family == "project_docs" and path.name in EXCLUDED_PROJECT_DOC_MATRIX_DOCS:
+                continue
+            model = _assess_doc(
+                repo_root=repo_root,
+                stage_name=f"source_doc_eval_{family}_{path.stem}",
+                output_model=DocumentAssessment,
+                context_payload={
+                    "run_id": run_id,
+                    "started_at": started_at,
+                    "eval_type": "sparse_input",
+                    "doc_family": family,
+                    "doc_name": path.name,
+                    "doc_content": path.read_text(encoding="utf-8"),
+                    "user_input": text,
+                },
+                guidance=load_agentic_prompt_lines("source_doc_eval_document"),
+            )
+            assessments.append(model)
+
+    overall = _summarize_assessments(eval_root, run_id, started_at, assessments)
+    manifest = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "eval_type": "sparse_input",
+        "repo_root": str(repo_root),
+        "eval_root": str(eval_root),
+        "agent_runtime_mode": agent_runtime_mode,
+        "dag_run_id": dag_result.run_id,
+        "dag_pipeline_dir": str(dag_result.pipeline_dir),
+        "artifacts": {
+            "source_docs_root": str(source_dst),
+            "project_docs_root": str(project_dst),
+            "source_docs_index": "indexes/source_docs_index.json",
+            "project_docs_index": "indexes/project_docs_index.json",
+            "quality_metrics": "metrics/quality.json",
+            "completeness_metrics": "metrics/completeness.json",
+        },
+        "copied_source_docs": source_copied,
+        "copied_project_docs": project_copied,
+        "overall_metrics": overall,
+    }
+    _write_json(eval_root / "eval_manifest.json", manifest)
+    return SourceDocMutationEvalResult(run_id=run_id, started_at=started_at, eval_root=eval_root, payload=manifest)
+
+
+def run_targeted_mutation_eval(*, repo_root: Path, base_text: str, mutation_text: str, agent_runtime_mode: str = "real", project_id: str = "eval_targeted_mutation", idea_id: str = "eval_targeted_mutation_001", pipeline_key_prefix: str = "eval_targeted_mutation", evals_root: Path | None = None, baseline_eval_root: Path | None = None) -> SourceDocMutationEvalResult:
+    run_id, started_at = _now_run_id()
+    resolved_evals_root = (evals_root or (repo_root / "evals")).resolve()
+    baseline = _resolve_targeted_mutation_baseline(evals_root=resolved_evals_root, baseline_eval_root=baseline_eval_root)
+
+    repo_source_root = repo_root / "ai_docs" / "context" / "source_docs"
+    repo_project_root = repo_root / "ai_docs" / "context" / "v2" / "project_docs" / "source_docs"
+    baseline_source_copied_to_repo = _copy_tree(baseline.source_docs_root, repo_source_root)
+    baseline_project_copied_to_repo = _copy_tree(baseline.project_docs_root, repo_project_root)
+
+    before_source = {name: _load_json(repo_source_root / name) for name in CANONICAL_SOURCE_DOCS}
+    before_project: dict[str, str] = {}
+    for name in PROJECT_DOC_TARGETS:
+        path = repo_project_root / name
+        before_project[name] = path.read_text(encoding="utf-8") if path.exists() else ""
+
+    mutation_request = build_source_doc_mutation_request(
+        project_id=project_id,
+        idea_id=idea_id,
+        repo_root=repo_root,
+        raw_text=mutation_text,
+        grounded_raw_text=mutation_text,
+        mode="ideation_mutation",
+        existing_source_doc_refs=_existing_source_refs(),
+        pipeline_key=f"{pipeline_key_prefix}_mutation",
+        agent_runtime_mode=agent_runtime_mode,
+    )
+    mutation_result = run_source_doc_mutation_dag(request=mutation_request)
+
+    eval_root = resolved_evals_root / run_id
+    source_dst, project_dst = _doc_paths(eval_root)
+    baseline_source_dst, baseline_project_dst = _baseline_doc_paths(eval_root)
+    baseline_source_copied = _copy_tree(baseline.source_docs_root, baseline_source_dst)
+    baseline_project_copied = _copy_tree(baseline.project_docs_root, baseline_project_dst)
+    source_copied = _copy_tree(repo_source_root, source_dst)
+    project_copied = _copy_tree(repo_project_root, project_dst)
+
+    assessments: list[MutationDocumentAssessment] = []
+    for family, names in (("source_docs", list(CANONICAL_SOURCE_DOCS)), ("project_docs", list(PROJECT_DOC_TARGETS))):
+        for name in names:
+            if family == "project_docs" and name in EXCLUDED_PROJECT_DOC_MATRIX_DOCS:
+                continue
+            before_payload = before_source[name] if family == "source_docs" else before_project[name]
+            after_path = (source_dst if family == "source_docs" else project_dst) / name
+            after_payload = after_path.read_text(encoding="utf-8")
+            model = _assess_doc(
+                repo_root=repo_root,
+                stage_name=f"source_doc_eval_mutation_{family}_{Path(name).stem}",
+                output_model=MutationDocumentAssessment,
+                context_payload={
+                    "run_id": run_id,
+                    "started_at": started_at,
+                    "eval_type": "targeted_mutation",
+                    "doc_family": family,
+                    "doc_name": name,
+                    "base_request": base_text,
+                    "mutation_request": mutation_text,
+                    "before_doc": before_payload,
+                    "after_doc": after_payload,
+                },
+                guidance=load_agentic_prompt_lines("source_doc_eval_targeted_mutation"),
+            )
+            assessments.append(model)
+
+    overall = _summarize_assessments(eval_root, run_id, started_at, assessments)
+    scored_project_docs = [a for a in assessments if a.doc_family == "project_docs" and a.doc_name not in EXCLUDED_PROJECT_DOC_MATRIX_DOCS]
+    scored_overall = [a for a in assessments if not (a.doc_family == "project_docs" and a.doc_name in EXCLUDED_PROJECT_DOC_MATRIX_DOCS)]
+    appropriate_matrix = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "metric": "appropriate_update",
+        "docs": [{"doc_name": a.doc_name, "doc_family": a.doc_family, "appropriate_update": a.appropriate_update} for a in assessments],
+        "excluded_project_docs": sorted(EXCLUDED_PROJECT_DOC_MATRIX_DOCS),
+        "source_docs": _mean([a.appropriate_update for a in assessments if a.doc_family == "source_docs"]),
+        "project_docs": _mean([a.appropriate_update for a in scored_project_docs]),
+        "overall": _mean([a.appropriate_update for a in scored_overall]),
+    }
+    thoroughness_matrix = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "metric": "thoroughness",
+        "docs": [{"doc_name": a.doc_name, "doc_family": a.doc_family, "thoroughness": a.thoroughness} for a in assessments],
+        "excluded_project_docs": sorted(EXCLUDED_PROJECT_DOC_MATRIX_DOCS),
+        "source_docs": _mean([a.thoroughness for a in assessments if a.doc_family == "source_docs"]),
+        "project_docs": _mean([a.thoroughness for a in scored_project_docs]),
+        "overall": _mean([a.thoroughness for a in scored_overall]),
+    }
+    accuracy_matrix = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "metric": "accuracy",
+        "formula": "average(appropriate_update, thoroughness)",
+        "docs": [{"doc_name": a.doc_name, "doc_family": a.doc_family, "accuracy": a.accuracy} for a in assessments],
+        "excluded_project_docs": sorted(EXCLUDED_PROJECT_DOC_MATRIX_DOCS),
+        "source_docs": _mean([a.accuracy for a in assessments if a.doc_family == "source_docs"]),
+        "project_docs": _mean([a.accuracy for a in scored_project_docs]),
+        "overall": _mean([a.accuracy for a in scored_overall]),
+    }
+    _write_json(eval_root / "matrices" / "appropriate_update.json", appropriate_matrix)
+    _write_json(eval_root / "matrices" / "thoroughness.json", thoroughness_matrix)
+    _write_json(eval_root / "matrices" / "accuracy.json", accuracy_matrix)
+
+    manifest = {
+        "run_id": run_id,
+        "started_at": started_at,
+        "eval_type": "targeted_mutation",
+        "repo_root": str(repo_root),
+        "eval_root": str(eval_root),
+        "agent_runtime_mode": agent_runtime_mode,
+        "baseline_eval_root": str(baseline.eval_root),
+        "baseline_eval_type": baseline.manifest.get("eval_type"),
+        "baseline_run_id": baseline.manifest.get("run_id"),
+        "baseline_started_at": baseline.manifest.get("started_at"),
+        "mutation_dag_run_id": mutation_result.run_id,
+        "mutation_dag_pipeline_dir": str(mutation_result.pipeline_dir),
+        "artifacts": {
+            "baseline_source_docs_root": str(baseline_source_dst),
+            "baseline_project_docs_root": str(baseline_project_dst),
+            "source_docs_root": str(source_dst),
+            "project_docs_root": str(project_dst),
+            "source_docs_index": "indexes/source_docs_index.json",
+            "project_docs_index": "indexes/project_docs_index.json",
+            "quality_metrics": "metrics/quality.json",
+            "completeness_metrics": "metrics/completeness.json",
+            "appropriate_update_matrix": "matrices/appropriate_update.json",
+            "thoroughness_matrix": "matrices/thoroughness.json",
+            "accuracy_matrix": "matrices/accuracy.json",
+        },
+        "baseline_copied_to_repo": {
+            "source_docs": baseline_source_copied_to_repo,
+            "project_docs": baseline_project_copied_to_repo,
+        },
+        "copied_baseline_source_docs": baseline_source_copied,
+        "copied_baseline_project_docs": baseline_project_copied,
+        "copied_source_docs": source_copied,
+        "copied_project_docs": project_copied,
+        "overall_metrics": overall,
+    }
+    _write_json(eval_root / "eval_manifest.json", manifest)
+    return SourceDocMutationEvalResult(run_id=run_id, started_at=started_at, eval_root=eval_root, payload=manifest)

devflow_engine/source_doc_mutation_worker.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Any
+
+from .stores.execution_store import ExecutionStore
+
+
+def enqueue_source_doc_mutation_task(
+    *,
+    store: ExecutionStore,
+    project_id: str | None,
+    source_run_id: str,
+    mutation_request: dict[str, Any],
+) -> dict[str, Any]:
+    queue_id = store.enqueue_source_doc_mutation_task(
+        project_id=project_id,
+        enqueue_run_id=source_run_id,
+        idea_id=str(mutation_request.get("idea_id") or "").strip() or None,
+        title=f"Source doc mutation for {str(mutation_request.get('idea_id') or 'unknown-idea')}",
+        mutation_request=mutation_request,
+    )
+    row = store.get_source_doc_mutation_queue_item(source_doc_mutation_queue_id=queue_id)
+    if row is None:
+        raise RuntimeError(f"source_doc_mutation_queue row missing after enqueue: {queue_id}")
+    return row

devflow_engine/source_docs_schema.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Literal
+
+SectionKind = Literal["string", "scalar_list", "object", "keyed_list"]
+MutationMode = Literal["replace", "merge", "remove"]
+
+
+@dataclass(frozen=True)
+class SourceDocSectionSchema:
+    kind: SectionKind
+    default: Any
+    topic_key_field: str | None = None
+    nested_scalar_lists: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class SourceDocSchema:
+    filename: str
+    sections: dict[str, SourceDocSectionSchema]
+
+
+def _clone(value: Any) -> Any:
+    return json.loads(json.dumps(value))
+
+
+SOURCE_DOC_SCHEMAS: dict[str, SourceDocSchema] = {
+    "product_brief.json": SourceDocSchema(
+        filename="product_brief.json",
+        sections={
+            "product_summary": SourceDocSectionSchema("string", ""),
+            "problem_statement": SourceDocSectionSchema("string", ""),
+            "target_users": SourceDocSectionSchema("scalar_list", []),
+            "goals": SourceDocSectionSchema("scalar_list", []),
+            "scope": SourceDocSectionSchema("object", {"in_scope": [], "out_of_scope": []}, nested_scalar_lists=("in_scope", "out_of_scope")),
+            "non_goals": SourceDocSectionSchema("scalar_list", []),
+            "constraints": SourceDocSectionSchema("scalar_list", []),
+            "success_criteria": SourceDocSectionSchema("scalar_list", []),
+            "assumptions": SourceDocSectionSchema("scalar_list", []),
+        },
+    ),
+    "user_workflows.json": SourceDocSchema(
+        filename="user_workflows.json",
+        sections={
+            "actors": SourceDocSectionSchema("scalar_list", []),
+            "primary_workflows": SourceDocSectionSchema("keyed_list", [], topic_key_field="workflow_key"),
+            "alternate_flows": SourceDocSectionSchema("keyed_list", [], topic_key_field="workflow_key"),
+            "edge_cases": SourceDocSectionSchema("scalar_list", []),
+            "workflow_assumptions": SourceDocSectionSchema("scalar_list", []),
+        },
+    ),
+    "domain_entities.json": SourceDocSchema(
+        filename="domain_entities.json",
+        sections={
+            "entities": SourceDocSectionSchema("keyed_list", [], topic_key_field="entity_key"),
+            "relationships": SourceDocSectionSchema("keyed_list", [], topic_key_field="relationship_key"),
+            "ownership_assumptions": SourceDocSectionSchema("scalar_list", []),
+        },
+    ),
+    "assumptions_registry.json": SourceDocSchema(
+        filename="assumptions_registry.json",
+        sections={
+            "assumptions": SourceDocSectionSchema("keyed_list", [], topic_key_field="assumption_id"),
+        },
+    ),
+}
+
+
+def source_doc_template_payloads() -> dict[str, dict[str, Any]]:
+    return {name: {section: _clone(spec.default) for section, spec in schema.sections.items()} for name, schema in SOURCE_DOC_SCHEMAS.items()}
+
+
+def _dedupe_strings(values: list[Any]) -> list[str]:
+    out: list[str] = []
+    for item in values:
+        if not isinstance(item, str):
+            continue
+        text = item.strip()
+        if text and text not in out:
+            out.append(text)
+    return out
+
+
+def normalize_source_doc_payload(doc_name: str, payload: Any) -> dict[str, Any]:
+    schema = SOURCE_DOC_SCHEMAS[doc_name]
+    raw = payload if isinstance(payload, dict) else {}
+    normalized: dict[str, Any] = {}
+    for section_key, section_schema in schema.sections.items():
+        value = raw.get(section_key, _clone(section_schema.default))
+        if section_schema.kind == "string":
+            normalized[section_key] = str(value).strip() if isinstance(value, str) else str(value).strip() if value not in (None, "") else ""
+        elif section_schema.kind == "scalar_list":
+            normalized[section_key] = _dedupe_strings(value if isinstance(value, list) else [])
+        elif section_schema.kind == "object":
+            source = value if isinstance(value, dict) else {}
+            section_value = _clone(section_schema.default)
+            for nested_key, nested_default in section_value.items():
+                nested_value = source.get(nested_key, nested_default)
+                if nested_key in section_schema.nested_scalar_lists:
+                    section_value[nested_key] = _dedupe_strings(nested_value if isinstance(nested_value, list) else [])
+                else:
+                    section_value[nested_key] = _clone(nested_value)
+            normalized[section_key] = section_value
+        elif section_schema.kind == "keyed_list":
+            key_field = str(section_schema.topic_key_field)
+            items: list[dict[str, Any]] = []
+            for item in value if isinstance(value, list) else []:
+                if not isinstance(item, dict):
+                    continue
+                topic_key = str(item.get(key_field) or "").strip()
+                if not topic_key or any(existing.get(key_field) == topic_key for existing in items):
+                    continue
+                clean = _clone(item)
+                clean[key_field] = topic_key
+                items.append(clean)
+            normalized[section_key] = items
+        else:
+            raise ValueError(f"Unsupported source doc section kind: {section_schema.kind}")
+    return normalized
+
+
+def apply_source_doc_mutation(
+    doc_name: str,
+    payload: Any,
+    *,
+    section_key: str,
+    topic_key: str | None = None,
+    value: Any = None,
+    mode: MutationMode = "merge",
+) -> tuple[dict[str, Any], bool]:
+    schema = SOURCE_DOC_SCHEMAS[doc_name]
+    if section_key not in schema.sections:
+        raise KeyError(f"Unknown section '{section_key}' for {doc_name}")
+    section_schema = schema.sections[section_key]
+    normalized = normalize_source_doc_payload(doc_name, payload)
+    before = json.dumps(normalized, sort_keys=True)
+
+    if section_schema.kind == "string":
+        if topic_key is not None:
+            raise ValueError(f"Section '{section_key}' does not support topic_key mutation")
+        normalized[section_key] = "" if mode == "remove" else str(value or "").strip()
+    elif section_schema.kind == "scalar_list":
+        current = list(normalized[section_key])
+        if topic_key is not None:
+            if mode == "remove":
+                normalized[section_key] = [item for item in current if item != topic_key]
+            else:
+                if value is not None:
+                    raise ValueError(f"Section '{section_key}' expects topic_key-only mutation or whole-list value")
+                normalized[section_key] = _dedupe_strings(current + [topic_key])
+        elif mode == "replace":
+            normalized[section_key] = _dedupe_strings(value if isinstance(value, list) else [])
+        elif mode == "remove":
+            removals = _dedupe_strings(value if isinstance(value, list) else [value])
+            normalized[section_key] = [item for item in current if item not in removals]
+        else:
+            additions = _dedupe_strings(value if isinstance(value, list) else [value])
+            normalized[section_key] = _dedupe_strings(current + additions)
+    elif section_schema.kind == "object":
+        current = dict(normalized[section_key])
+        if topic_key is None:
+            if mode == "replace":
+                current = normalize_source_doc_payload(doc_name, {section_key: value}).get(section_key, _clone(section_schema.default))
+            elif isinstance(value, dict):
+                merged = current | value
+                current = normalize_source_doc_payload(doc_name, {section_key: merged}).get(section_key, _clone(section_schema.default))
+            else:
+                raise ValueError(f"Section '{section_key}' expects dict value when mutating without topic_key")
+        elif topic_key in section_schema.nested_scalar_lists:
+            existing = list(current.get(topic_key) or [])
+            if mode == "replace":
+                current[topic_key] = _dedupe_strings(value if isinstance(value, list) else [])
+            elif mode == "remove":
+                removals = _dedupe_strings(value if isinstance(value, list) else [value])
+                current[topic_key] = [item for item in existing if item not in removals]
+            else:
+                additions = _dedupe_strings(value if isinstance(value, list) else [value])
+                current[topic_key] = _dedupe_strings(existing + additions)
+        else:
+            raise KeyError(f"Unknown topic '{topic_key}' for section '{section_key}'")
+        normalized[section_key] = current
+    elif section_schema.kind == "keyed_list":
+        key_field = str(section_schema.topic_key_field)
+        current = list(normalized[section_key])
+        if not topic_key:
+            raise ValueError(f"Section '{section_key}' requires topic_key mutation")
+        survivors = [item for item in current if str(item.get(key_field) or "") != topic_key]
+        if mode == "remove":
+            normalized[section_key] = survivors
+        else:
+            incoming = value if isinstance(value, dict) else {}
+            previous = next((item for item in current if str(item.get(key_field) or "") == topic_key), {})
+            if mode == "replace":
+                merged = dict(incoming)
+            else:
+                merged = dict(previous)
+                merged.update(incoming)
+            merged[key_field] = topic_key
+            normalized[section_key] = survivors + [merged]
+    else:
+        raise ValueError(f"Unsupported source doc section kind: {section_schema.kind}")
+
+    normalized = normalize_source_doc_payload(doc_name, normalized)
+    after = json.dumps(normalized, sort_keys=True)
+    return normalized, before != after