devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/idea/ideation_enrichment.py

@@ -0,0 +1,355 @@
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+from ..agentic_runtime import _load_llm_cli_config
+from ..llm.cli_stream import run_streaming
+from ..orchestration import process_one_task
+from ..stores.execution_store import ExecutionStore
+
+IDEATION_ENRICHMENT_QUEUE_NAME = "idea.ideation_enrichment"
+IDEATION_ENRICHMENT_TASK_KIND = "idea.ideation.enrichment"
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _load_json_dict(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _idea_file_path(repo_root: Path, *, idea_id: str) -> Path:
+    return repo_root / ".devflow" / "ideas" / idea_id / "idea.json"
+
+
+def ideation_enrichment_status_path(repo_root: Path, *, idea_id: str) -> Path:
+    return repo_root / ".devflow" / "ideas" / idea_id / "ideation_enrichment.json"
+
+
+def load_ideation_enrichment_status(repo_root: Path, *, idea_id: str) -> dict[str, Any]:
+    return _load_json_dict(ideation_enrichment_status_path(repo_root, idea_id=idea_id))
+
+
+def _sync_idea_enrichment(repo_root: Path, *, idea_id: str, snapshot: dict[str, Any]) -> None:
+    idea_path = _idea_file_path(repo_root, idea_id=idea_id)
+    idea_payload = _load_json_dict(idea_path) if idea_path.exists() else {"idea_id": idea_id}
+    idea_payload["enrichment"] = dict(snapshot)
+    _write_json(idea_path, idea_payload)
+
+
+def persist_ideation_enrichment_progress(
+    *,
+    repo_root: Path,
+    idea_id: str,
+    project_id: str | None,
+    pipeline_key: str,
+    task_id: str | None,
+    status: str,
+    stage: str,
+    progress_pct: int,
+    summary: str,
+    prompt_path: str | None = None,
+    stream_path: str | None = None,
+    terminal_path: str | None = None,
+    log_path: str | None = None,
+    contract: dict[str, Any] | None = None,
+    response_guidance: dict[str, Any] | None = None,
+    result_shape: dict[str, Any] | None = None,
+    error: str | None = None,
+) -> dict[str, Any]:
+    now = int(time.time())
+    status_path = ideation_enrichment_status_path(repo_root, idea_id=idea_id)
+    existing = _load_json_dict(status_path)
+    snapshot: dict[str, Any] = {
+        **existing,
+        "kind": "idea_ideation_enrichment_status.v1",
+        "idea_id": idea_id,
+        "project_id": project_id,
+        "pipeline_key": pipeline_key,
+        "task_id": task_id,
+        "status": status,
+        "stage": stage,
+        "progress_pct": max(0, min(int(progress_pct), 100)),
+        "summary": summary,
+        "inspect_path": str(status_path),
+        "updated_at": now,
+    }
+    if existing.get("queued_at") is None and status == "queued":
+        snapshot["queued_at"] = now
+    if existing.get("started_at") is None and status == "running":
+        snapshot["started_at"] = now
+    if status in {"completed", "failed"}:
+        snapshot["completed_at"] = now
+    if prompt_path:
+        snapshot["prompt_path"] = prompt_path
+    if stream_path:
+        snapshot["stream_path"] = stream_path
+    if terminal_path:
+        snapshot["terminal_path"] = terminal_path
+    if log_path:
+        snapshot["log_path"] = log_path
+    if contract is not None:
+        snapshot["contract"] = dict(contract)
+    if response_guidance is not None:
+        snapshot["response_guidance"] = dict(response_guidance)
+    if result_shape is not None:
+        snapshot["result_shape"] = dict(result_shape)
+    if error:
+        snapshot["error"] = error
+    _write_json(status_path, snapshot)
+    _sync_idea_enrichment(repo_root, idea_id=idea_id, snapshot=snapshot)
+    return snapshot
+
+
+def enqueue_ideation_enrichment_task(
+    *,
+    store: ExecutionStore,
+    project_id: str | None,
+    repo_root: Path,
+    idea_id: str,
+    pipeline_key: str,
+    prompt_payload: dict[str, Any],
+    prompt_path: Path,
+    response_guidance: dict[str, Any],
+    contract: dict[str, Any],
+    source_run_id: str | None = None,
+) -> dict[str, Any]:
+    task = store.enqueue_task(
+        project_id=project_id,
+        queue_name=IDEATION_ENRICHMENT_QUEUE_NAME,
+        task_kind=IDEATION_ENRICHMENT_TASK_KIND,
+        idempotency_key=f"{idea_id}:{pipeline_key}:ideation_enrichment",
+        source_run_id=source_run_id,
+        max_attempts=1,
+        input_payload={
+            "idea_id": idea_id,
+            "project_id": project_id,
+            "pipeline_key": pipeline_key,
+            "prompt_payload": prompt_payload,
+            "prompt_path": str(prompt_path),
+            "response_guidance": response_guidance,
+            "contract": contract,
+        },
+        context={"workflow": "idea_ideation_enrichment", "repo_root": str(repo_root)},
+    )
+    snapshot = persist_ideation_enrichment_progress(
+        repo_root=repo_root,
+        idea_id=idea_id,
+        project_id=project_id,
+        pipeline_key=pipeline_key,
+        task_id=str(task.get("task_id") or ""),
+        status="queued",
+        stage="queued",
+        progress_pct=5,
+        summary="Background ideation enrichment queued.",
+        prompt_path=str(prompt_path),
+        contract=contract,
+        response_guidance=response_guidance,
+    )
+    return {"task": task, "status": snapshot}
+
+
+def _extract_json_object(raw_text: str) -> dict[str, Any]:
+    candidate = raw_text.strip()
+    if "```" in candidate:
+        for chunk in candidate.split("```"):
+            stripped = chunk.strip()
+            if stripped.startswith("json"):
+                stripped = stripped[4:].strip()
+            if stripped.startswith("{"):
+                candidate = stripped
+                break
+    parsed = json.loads(candidate)
+    if not isinstance(parsed, dict):
+        raise RuntimeError("ideation enrichment worker expected a JSON object result")
+    return parsed
+
+
+def handle_ideation_enrichment_task(store: ExecutionStore, repo_root: Path, task: dict[str, Any]) -> dict[str, Any]:
+    payload = dict(task.get("input") or {})
+    task_id = str(task["task_id"])
+    idea_id = str(payload.get("idea_id") or "")
+    project_id = str(payload.get("project_id") or "").strip() or None
+    pipeline_key = str(payload.get("pipeline_key") or "")
+    prompt_payload = dict(payload.get("prompt_payload") or {})
+    prompt_path = Path(str(payload.get("prompt_path") or ""))
+    response_guidance = dict(payload.get("response_guidance") or {})
+    contract = dict(payload.get("contract") or {})
+    pipeline_root = prompt_path.parent
+    stream_path = pipeline_root / "ideation_agent_loop_stream.json"
+    terminal_path = pipeline_root / "ideation_agent_loop_terminal.json"
+
+    store.record_task_step(task_id=task_id, step_name="ideation_enrichment.claim", status="started", payload={"idea_id": idea_id, "pipeline_key": pipeline_key})
+    persist_ideation_enrichment_progress(
+        repo_root=repo_root,
+        idea_id=idea_id,
+        project_id=project_id,
+        pipeline_key=pipeline_key,
+        task_id=task_id,
+        status="running",
+        stage="starting",
+        progress_pct=15,
+        summary="Background ideation enrichment started.",
+        prompt_path=str(prompt_path),
+        contract=contract,
+        response_guidance=response_guidance,
+    )
+    _write_json(prompt_path, prompt_payload)
+    store.emit_task_message(task_id=task_id, message_kind="partial", stream="ideation_enrichment", payload={"status": "running", "stage": "prompt_written", "prompt_path": str(prompt_path)})
+    persist_ideation_enrichment_progress(
+        repo_root=repo_root,
+        idea_id=idea_id,
+        project_id=project_id,
+        pipeline_key=pipeline_key,
+        task_id=task_id,
+        status="running",
+        stage="prompt_written",
+        progress_pct=25,
+        summary="Background ideation enrichment prompt persisted.",
+        prompt_path=str(prompt_path),
+        contract=contract,
+        response_guidance=response_guidance,
+    )
+
+    base_cmd, delivery = _load_llm_cli_config()
+    store.record_task_step(task_id=task_id, step_name="ideation_enrichment.run", status="started", payload={"delivery": delivery, "base_cmd": base_cmd})
+    persist_ideation_enrichment_progress(
+        repo_root=repo_root,
+        idea_id=idea_id,
+        project_id=project_id,
+        pipeline_key=pipeline_key,
+        task_id=task_id,
+        status="running",
+        stage="agent_running",
+        progress_pct=45,
+        summary="Background ideation enrichment is running.",
+        prompt_path=str(prompt_path),
+        contract=contract,
+        response_guidance=response_guidance,
+    )
+    stream_result = run_streaming(
+        provider="cli",
+        base_cmd=base_cmd,
+        delivery=delivery,
+        prompt=json.dumps(prompt_payload, indent=2, sort_keys=True),
+        cwd=repo_root,
+        timeout_s=900,
+    )
+    stream_payload = {
+        "base_cmd": base_cmd,
+        "delivery": delivery,
+        "session_id": stream_result.session_id,
+        "returncode": stream_result.returncode,
+        "ok": stream_result.ok,
+        "log_path": str(stream_result.log_path),
+        "stdout": stream_result.stdout,
+        "stderr": stream_result.stderr,
+    }
+    _write_json(stream_path, stream_payload)
+    store.emit_task_message(task_id=task_id, message_kind="partial", stream="ideation_enrichment", payload={"status": "running", "stage": "stream_persisted", "stream_path": str(stream_path), "session_id": stream_result.session_id})
+    if not stream_result.ok:
+        failure = {
+            "status": "failed",
+            "idea_id": idea_id,
+            "task_id": task_id,
+            "pipeline_key": pipeline_key,
+            "session_id": stream_result.session_id,
+            "log_path": str(stream_result.log_path),
+            "returncode": stream_result.returncode,
+            "stdout": stream_result.stdout,
+            "stderr": stream_result.stderr,
+            "response_guidance": response_guidance,
+        }
+        _write_json(terminal_path, failure)
+        persist_ideation_enrichment_progress(
+            repo_root=repo_root,
+            idea_id=idea_id,
+            project_id=project_id,
+            pipeline_key=pipeline_key,
+            task_id=task_id,
+            status="failed",
+            stage="failed",
+            progress_pct=100,
+            summary="Background ideation enrichment failed.",
+            prompt_path=str(prompt_path),
+            stream_path=str(stream_path),
+            terminal_path=str(terminal_path),
+            log_path=str(stream_result.log_path),
+            contract=contract,
+            response_guidance=response_guidance,
+            error=f"returncode={stream_result.returncode}",
+        )
+        store.record_task_step(task_id=task_id, step_name="ideation_enrichment.run", status="failed", payload={"returncode": stream_result.returncode, "log_path": str(stream_result.log_path)})
+        raise RuntimeError(f"ideation enrichment worker failed with returncode={stream_result.returncode}")
+
+    result_shape = _extract_json_object(stream_result.stdout)
+    terminal = {
+        "status": "completed",
+        "idea_id": idea_id,
+        "task_id": task_id,
+        "pipeline_key": pipeline_key,
+        "session_id": stream_result.session_id,
+        "log_path": str(stream_result.log_path),
+        "returncode": stream_result.returncode,
+        "response_guidance": response_guidance,
+        "result_shape": result_shape,
+        "agent_loop_stream": stream_payload,
+    }
+    _write_json(terminal_path, terminal)
+    snapshot = persist_ideation_enrichment_progress(
+        repo_root=repo_root,
+        idea_id=idea_id,
+        project_id=project_id,
+        pipeline_key=pipeline_key,
+        task_id=task_id,
+        status="completed",
+        stage="completed",
+        progress_pct=100,
+        summary=str(result_shape.get("summary") or "Background ideation enrichment completed."),
+        prompt_path=str(prompt_path),
+        stream_path=str(stream_path),
+        terminal_path=str(terminal_path),
+        log_path=str(stream_result.log_path),
+        contract=contract,
+        response_guidance=response_guidance,
+        result_shape=result_shape,
+    )
+    response = {
+        "idea_id": idea_id,
+        "pipeline_key": pipeline_key,
+        "status": "completed",
+        "status_path": str(ideation_enrichment_status_path(repo_root, idea_id=idea_id)),
+        "terminal_path": str(terminal_path),
+        "stream_path": str(stream_path),
+        "result_shape": result_shape,
+        "snapshot": snapshot,
+    }
+    store.emit_task_message(task_id=task_id, message_kind="result", stream="ideation_enrichment", payload=response)
+    store.record_task_step(task_id=task_id, step_name="ideation_enrichment.run", status="completed", payload={"terminal_path": str(terminal_path), "stream_path": str(stream_path)})
+    return response
+
+
+def process_ideation_enrichment_once(
+    *,
+    store: ExecutionStore,
+    repo_root: Path,
+    worker_id: str = "idea-ideation-enrichment",
+):
+    return process_one_task(
+        store=store,
+        repo_root=repo_root,
+        queue_name=IDEATION_ENRICHMENT_QUEUE_NAME,
+        worker_id=worker_id,
+        handlers={IDEATION_ENRICHMENT_TASK_KIND: handle_ideation_enrichment_task},
+    )

devflow_engine/idea/ideation_enrichment_worker.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from devflow_engine.idea.devin_chat_dag import run_contract_enrichment_worker
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run Devin ideation contract enrichment in the background.")
+    parser.add_argument("--repo-root", required=True)
+    parser.add_argument("--idea-id", required=True)
+    args = parser.parse_args()
+    run_contract_enrichment_worker(repo_root=Path(args.repo_root).expanduser().resolve(), idea_id=str(args.idea_id))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

devflow_engine/idea/paths.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class IdeaPaths:
+    repo_root: Path
+    devflow_dir: Path
+    ideas_root: Path
+    idea_dir: Path
+    analysis_root: Path
+    drafts_root: Path
+
+
+def get_idea_paths(repo_root: Path, *, idea_id: str) -> IdeaPaths:
+    devflow_dir = repo_root / ".devflow"
+    ideas_root = devflow_dir / "ideas"
+    idea_dir = ideas_root / idea_id
+    return IdeaPaths(
+        repo_root=repo_root,
+        devflow_dir=devflow_dir,
+        ideas_root=ideas_root,
+        idea_dir=idea_dir,
+        analysis_root=idea_dir / "analysis",
+        drafts_root=idea_dir / "drafts",
+    )

devflow_engine/idea/promote.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from .drafts import dest_filename_for_draft, parse_locked_header
+from .paths import get_idea_paths
+
+
+@dataclass(frozen=True)
+class PromoteOp:
+    kind: str  # create|update
+    src: Path
+    dest: Path
+
+
+def _load_manifest(repo_root: Path, *, idea_id: str, draft_set_id: str) -> dict:
+    paths = get_idea_paths(repo_root, idea_id=idea_id)
+    p = paths.drafts_root / draft_set_id / "manifest.json"
+    return json.loads(p.read_text(encoding="utf-8"))
+
+
+def plan_promotion(*, repo_root: Path, idea_id: str, draft_set_id: str, dest_root: Path) -> list[PromoteOp]:
+    manifest = _load_manifest(repo_root, idea_id=idea_id, draft_set_id=draft_set_id)
+    draft_paths = manifest.get("draft_paths") or []
+
+    ops: list[PromoteOp] = []
+    for rel in draft_paths:
+        src = repo_root / rel
+        header = parse_locked_header(src.read_text(encoding="utf-8", errors="ignore"))
+        dest_name = dest_filename_for_draft(src, header=header)
+        dest = dest_root / dest_name
+        kind = "update" if dest.exists() else "create"
+        ops.append(PromoteOp(kind=kind, src=src, dest=dest))
+
+    ops.sort(key=lambda o: (o.dest.as_posix(), o.src.as_posix()))
+    return ops
+
+
+def apply_promotion(*, ops: list[PromoteOp], force: bool) -> tuple[bool, list[str]]:
+    messages: list[str] = []
+    for op in ops:
+        if op.dest.exists() and not force:
+            messages.append(f"conflict: destination exists: {op.dest}")
+            return False, messages
+
+    for op in ops:
+        op.dest.parent.mkdir(parents=True, exist_ok=True)
+        op.dest.write_text(op.src.read_text(encoding="utf-8"), encoding="utf-8")
+        messages.append(f"{op.kind}: {op.dest}")
+
+    return True, messages

devflow_engine/idea/redaction.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+# Intentionally lightweight, heuristic redaction.
+_SECRET_PATTERNS: list[re.Pattern[str]] = [
+    re.compile(r"sk_(live|test)_[A-Za-z0-9]{10,}"),
+    re.compile(r"AKIA[0-9A-Z]{16}"),
+]
+
+
+@dataclass(frozen=True)
+class RedactionResult:
+    text: str
+    redacted: bool
+
+
+def redact_text(text: str) -> RedactionResult:
+    redacted_any = False
+    out = text
+    for pat in _SECRET_PATTERNS:
+        if pat.search(out):
+            redacted_any = True
+            out = pat.sub("***REDACTED***", out)
+    return RedactionResult(text=out, redacted=redacted_any)