devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/source_docs_updater.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+
+from .orchestration import process_one_task
+from .source_doc_assumptions import (
+    default_assumptions_registry_payload,
+    normalize_assumptions_registry,
+    reconcile_assumptions_registry,
+)
+from .source_docs_schema import apply_source_doc_mutation, normalize_source_doc_payload, source_doc_template_payloads
+from .stores.execution_store import ExecutionStore
+
+SOURCE_DOC_UPDATE_QUEUE_NAME = "project.source_docs"
+SOURCE_DOC_UPDATE_TASK_KIND = "project.source_docs.update"
+
+SOURCE_DOC_TEMPLATE_PAYLOADS: dict[str, dict[str, Any]] = source_doc_template_payloads()
+# Back-compat alias while callers migrate.
+SOURCE_DOC_TEMPLATES = SOURCE_DOC_TEMPLATE_PAYLOADS
+
+
+@dataclass(frozen=True)
+class SourceDocProjectPaths:
+    repo_root: Path
+    source_docs_dir: Path
+    project_docs_dir: Path
+    state_path: Path
+
+
+@dataclass(frozen=True)
+class SourceDocSyncPlan:
+    changed_docs: list[str]
+    source_hashes: dict[str, str]
+    previous_hashes: dict[str, str]
+    state_path: Path
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.changed_docs)
+
+
+@dataclass(frozen=True)
+class SourceDocUpdateResult:
+    changed_docs: list[str]
+    updated_outputs: list[str]
+    state_path: Path
+    project_docs_dir: Path
+
+
+@dataclass(frozen=True)
+class QueuedSourceDocUpdate:
+    enqueued: bool
+    task: dict[str, Any] | None
+    plan: SourceDocSyncPlan
+
+
+def get_source_doc_paths(repo_root: Path) -> SourceDocProjectPaths:
+    repo_root = repo_root.expanduser().resolve()
+    source_docs_dir = repo_root / "ai_docs" / "context" / "source_docs"
+    project_docs_dir = repo_root / "ai_docs" / "context" / "project_docs"
+    state_path = project_docs_dir / ".source_doc_updater_state.json"
+    return SourceDocProjectPaths(
+        repo_root=repo_root,
+        source_docs_dir=source_docs_dir,
+        project_docs_dir=project_docs_dir,
+        state_path=state_path,
+    )
+
+
+def ensure_source_doc_scaffold(repo_root: Path) -> SourceDocProjectPaths:
+    paths = get_source_doc_paths(repo_root)
+    paths.source_docs_dir.mkdir(parents=True, exist_ok=True)
+    paths.project_docs_dir.mkdir(parents=True, exist_ok=True)
+    for filename, payload in source_doc_template_payloads().items():
+        path = paths.source_docs_dir / filename
+        if not path.exists():
+            initial_payload = default_assumptions_registry_payload() if filename == "assumptions_registry.json" else payload
+            path.write_text(json.dumps(initial_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+            continue
+        if filename == "assumptions_registry.json":
+            normalized = normalize_assumptions_registry(_load_json(path))
+        else:
+            normalized = normalize_source_doc_payload(filename, _load_json(path))
+        path.write_text(json.dumps(normalized, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return paths
+
+
+def _read_updater_state(state_path: Path) -> dict[str, Any]:
+    try:
+        raw = json.loads(state_path.read_text(encoding="utf-8"))
+    except Exception:
+        return {"source_hashes": {}, "updated_at": None, "updated_outputs": []}
+    if not isinstance(raw, dict):
+        return {"source_hashes": {}, "updated_at": None, "updated_outputs": []}
+    hashes = raw.get("source_hashes")
+    if not isinstance(hashes, dict):
+        hashes = {}
+    return {
+        "source_hashes": {str(k): str(v) for k, v in hashes.items()},
+        "updated_at": raw.get("updated_at"),
+        "updated_outputs": [str(item) for item in (raw.get("updated_outputs") or []) if isinstance(item, str)],
+    }
+
+
+def _hash_file(path: Path) -> str:
+    return sha256(path.read_bytes()).hexdigest()
+
+
+def detect_source_doc_changes(repo_root: Path) -> SourceDocSyncPlan:
+    paths = ensure_source_doc_scaffold(repo_root)
+    previous_state = _read_updater_state(paths.state_path)
+    previous_hashes = dict(previous_state.get("source_hashes") or {})
+
+    source_hashes: dict[str, str] = {}
+    for path in sorted(paths.source_docs_dir.glob("*.json")):
+        source_hashes[path.name] = _hash_file(path)
+
+    changed_docs = [name for name, digest in source_hashes.items() if previous_hashes.get(name) != digest]
+    return SourceDocSyncPlan(
+        changed_docs=changed_docs,
+        source_hashes=source_hashes,
+        previous_hashes=previous_hashes,
+        state_path=paths.state_path,
+    )
+
+
+def _load_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def mutate_source_doc(
+    repo_root: Path,
+    *,
+    doc_name: str,
+    section_key: str,
+    topic_key: str | None = None,
+    value: Any = None,
+    mode: str = "merge",
+) -> tuple[dict[str, Any], bool]:
+    paths = ensure_source_doc_scaffold(repo_root)
+    source_path = paths.source_docs_dir / doc_name
+    payload = _load_json(source_path) if source_path.exists() else SOURCE_DOC_TEMPLATE_PAYLOADS[doc_name]
+    updated, changed = apply_source_doc_mutation(
+        doc_name,
+        payload,
+        section_key=section_key,
+        topic_key=topic_key,
+        value=value,
+        mode=mode,
+    )
+    if changed:
+        source_path.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return updated, changed
+
+
+def _render_markdown(doc_name: str, payload: Any) -> str:
+    title = doc_name.replace("_", " ").replace(".json", "").title()
+    pretty_json = json.dumps(payload, indent=2, sort_keys=True)
+    summary_lines = [f"# {title}", "", f"Derived from `ai_docs/context/source_docs/{doc_name}`.", ""]
+    if isinstance(payload, dict):
+        for key, value in payload.items():
+            summary_lines.append(f"## {str(key).replace('_', ' ').title()}")
+            if isinstance(value, list):
+                if value:
+                    for item in value:
+                        summary_lines.append(f"- {json.dumps(item, sort_keys=True) if isinstance(item, (dict, list)) else item}")
+                else:
+                    summary_lines.append("- _empty_")
+            elif isinstance(value, dict):
+                if value:
+                    for sub_key, sub_value in value.items():
+                        rendered = json.dumps(sub_value, sort_keys=True) if isinstance(sub_value, (dict, list)) else sub_value
+                        summary_lines.append(f"- **{sub_key}**: {rendered}")
+                else:
+                    summary_lines.append("- _empty_")
+            else:
+                summary_lines.append(str(value) if value not in (None, "") else "_empty_")
+            summary_lines.append("")
+    summary_lines.extend(["## Canonical JSON", "", "```json", pretty_json, "```", ""])
+    return "\n".join(summary_lines)
+
+
+def apply_source_doc_updates(repo_root: Path, *, changed_docs: list[str] | None = None) -> SourceDocUpdateResult:
+    paths = ensure_source_doc_scaffold(repo_root)
+    plan = detect_source_doc_changes(repo_root)
+    docs_to_update = sorted(set(changed_docs if changed_docs is not None else plan.changed_docs))
+    if changed_docs is not None:
+        docs_to_update = [name for name in docs_to_update if (paths.source_docs_dir / name).exists()]
+
+    source_docs_by_name = {path.name: _load_json(path) for path in sorted(paths.source_docs_dir.glob("*.json"))}
+    registry_payload = normalize_assumptions_registry(source_docs_by_name.get("assumptions_registry.json"))
+    reconciled_registry, registry_changed = reconcile_assumptions_registry(
+        registry_payload=registry_payload,
+        source_docs_by_name=source_docs_by_name,
+        changed_docs=docs_to_update,
+    )
+    if registry_changed or reconciled_registry != source_docs_by_name.get("assumptions_registry.json"):
+        registry_path = paths.source_docs_dir / "assumptions_registry.json"
+        registry_path.write_text(json.dumps(reconciled_registry, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        source_docs_by_name["assumptions_registry.json"] = reconciled_registry
+        if "assumptions_registry.json" not in docs_to_update:
+            docs_to_update.append("assumptions_registry.json")
+    docs_to_update = sorted(set(docs_to_update))
+
+    updated_outputs: list[str] = []
+    for doc_name in docs_to_update:
+        source_path = paths.source_docs_dir / doc_name
+        if doc_name == "assumptions_registry.json":
+            payload = normalize_assumptions_registry(_load_json(source_path))
+        else:
+            payload = normalize_source_doc_payload(doc_name, _load_json(source_path))
+        source_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        out_name = doc_name.replace(".json", ".md")
+        out_path = paths.project_docs_dir / out_name
+        out_path.write_text(_render_markdown(doc_name, payload), encoding="utf-8")
+        updated_outputs.append(str(out_path.relative_to(paths.repo_root)))
+
+    if docs_to_update:
+        current_hashes = {path.name: _hash_file(path) for path in sorted(paths.source_docs_dir.glob("*.json"))}
+        index_lines = ["# Source Docs Sync Index", "", "This directory contains project docs derived from canonical source docs.", ""]
+        for doc_name in sorted(current_hashes):
+            md_name = doc_name.replace(".json", ".md")
+            marker = "updated" if doc_name in docs_to_update else "current"
+            index_lines.append(f"- `{md_name}` ← `{doc_name}` ({marker})")
+        index_lines.append("")
+        index_lines.append("## Source Hashes")
+        index_lines.append("")
+        for doc_name, digest in sorted(current_hashes.items()):
+            index_lines.append(f"- `{doc_name}`: `{digest}`")
+        index_path = paths.project_docs_dir / "source_docs_index.md"
+        index_path.write_text("\n".join(index_lines) + "\n", encoding="utf-8")
+        updated_outputs.append(str(index_path.relative_to(paths.repo_root)))
+    else:
+        current_hashes = {path.name: _hash_file(path) for path in sorted(paths.source_docs_dir.glob("*.json"))}
+
+    state_payload = {
+        "source_hashes": current_hashes,
+        "updated_at": int(time.time()),
+        "updated_outputs": updated_outputs,
+        "changed_docs": docs_to_update,
+    }
+    paths.state_path.write_text(json.dumps(state_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return SourceDocUpdateResult(
+        changed_docs=docs_to_update,
+        updated_outputs=updated_outputs,
+        state_path=paths.state_path,
+        project_docs_dir=paths.project_docs_dir,
+    )
+
+
+def enqueue_source_doc_update_if_needed(
+    *,
+    store: ExecutionStore,
+    project_id: str | None,
+    repo_root: Path,
+    source_run_id: str | None = None,
+) -> QueuedSourceDocUpdate:
+    plan = detect_source_doc_changes(repo_root)
+    if not plan.has_changes:
+        return QueuedSourceDocUpdate(enqueued=False, task=None, plan=plan)
+    if project_id:
+        with store._connect() as conn:
+            store._ensure_project_row(conn, project_id=project_id, repo_root=repo_root)
+    digest_material = [f"{name}:{plan.source_hashes[name]}" for name in sorted(plan.changed_docs)]
+    task = store.enqueue_task(
+        project_id=project_id,
+        queue_name=SOURCE_DOC_UPDATE_QUEUE_NAME,
+        task_kind=SOURCE_DOC_UPDATE_TASK_KIND,
+        idempotency_key="|".join(digest_material),
+        source_run_id=source_run_id,
+        input_payload={
+            "changed_docs": plan.changed_docs,
+            "source_hashes": {name: plan.source_hashes[name] for name in plan.changed_docs},
+        },
+        context={"workflow": "source_docs_to_project_docs", "repo_root": str(repo_root)},
+    )
+    return QueuedSourceDocUpdate(enqueued=True, task=task, plan=plan)
+
+
+def handle_source_doc_update_task(store: ExecutionStore, repo_root: Path, task: dict[str, Any]) -> dict[str, Any]:
+    payload = dict(task.get("input") or {})
+    task_id = str(task["task_id"])
+    changed_docs = [str(item) for item in (payload.get("changed_docs") or []) if isinstance(item, str)]
+    store.record_task_step(task_id=task_id, step_name="source_docs.detect", status="started", payload={"changed_docs": changed_docs})
+    result = apply_source_doc_updates(repo_root, changed_docs=changed_docs)
+    response = {
+        "changed_docs": result.changed_docs,
+        "updated_outputs": result.updated_outputs,
+        "state_path": str(result.state_path),
+        "project_docs_dir": str(result.project_docs_dir),
+    }
+    store.emit_task_message(task_id=task_id, message_kind="result", stream="project_docs", payload=response)
+    store.record_task_step(task_id=task_id, step_name="source_docs.render", status="completed", payload=response)
+    return response
+
+
+def process_source_doc_update_once(*, store: ExecutionStore, repo_root: Path, worker_id: str = "source-doc-updater"):
+    return process_one_task(
+        store=store,
+        repo_root=repo_root,
+        queue_name=SOURCE_DOC_UPDATE_QUEUE_NAME,
+        worker_id=worker_id,
+        handlers={SOURCE_DOC_UPDATE_TASK_KIND: handle_source_doc_update_task},
+    )

devflow_engine/source_scope/__init__.py

@@ -0,0 +1,15 @@
+from .dag import (
+    DAG_ID,
+    SourceDocsToScopesDagResult,
+    SourceToScopeDagResult,
+    run_source_docs_to_scopes_dag,
+    run_source_to_scope_dag,
+)
+
+__all__ = [
+    "DAG_ID",
+    "SourceDocsToScopesDagResult",
+    "SourceToScopeDagResult",
+    "run_source_docs_to_scopes_dag",
+    "run_source_to_scope_dag",
+]

devflow_engine/source_scope/agentic.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from ..agentic_prompts import load_agentic_prompt_lines
+from ..agentic_runtime import AgentRunEnvelope, run_agent_step
+
+
+SOURCE_SCOPE_DOCTRINE = load_agentic_prompt_lines("source_scope_doctrine")
+
+
+def run_source_scope_agent_step(
+    *,
+    repo_root: Path,
+    stage_name: str,
+    output_model: type[BaseModel],
+    context_payload: dict[str, Any],
+    guidance: list[str],
+    timeout_seconds: int | None = None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    return run_agent_step(
+        repo_root=repo_root,
+        stage_name=f"source_scope_{stage_name}",
+        output_model=output_model,
+        context_payload=context_payload,
+        guidance=SOURCE_SCOPE_DOCTRINE + guidance,
+        timeout_seconds=timeout_seconds,
+    )
+
+
+def persist_agent_run(*, pipeline_root: Path, node_id: str, envelope: AgentRunEnvelope) -> Path:
+    path = pipeline_root / f"{node_id}_agent_run.json"
+    payload = {
+        "node_id": node_id,
+        "prompt": envelope.prompt,
+        "response": envelope.response,
+        "raw_stdout": envelope.raw_stdout,
+        "raw_stderr": envelope.raw_stderr,
+    }
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return path