devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/idea/agentic.py

@@ -0,0 +1,465 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from ..agentic_prompts import load_agentic_prompt_lines, load_agentic_prompt_text
+from ..agentic_runtime import (
+    AgentRunEnvelope,
+    _build_artifact_reference_prompt,
+    _build_prompt,
+    _extract_json_hardened,
+    _persist_large_context_artifact,
+)
+from devflow_engine.devin2.pi_runner import run_devin2_pi_agent
+from ..llm.invoke import LlmInvocationRequest, invoke_llm
+from .traditional_stories import _global_devflow_dir, _read_toml
+from .response_mode import current_response_mode_label
+from ..llm.provider_api import (
+    ANTHROPIC_MODEL_DEFAULT,
+    AnthropicApiSettings as _AnthropicSettings,
+    anthropic_messages_create as _shared_anthropic_messages_create,
+    anthropic_response_text,
+    load_anthropic_api_settings,
+    read_anthropic_api_key,
+)
+from .repo_tools import (
+    INSIGHT_REPO_TOOL_DISPATCH,
+    INSIGHT_REPO_TOOL_SPECS,
+    REPO_TOOL_DISPATCH,
+    REPO_TOOL_SPECS,
+    activate_repo_tools,
+)
+
+
+IDEATION_RESPONSE_DOCTRINE = load_agentic_prompt_lines("idea_response_doctrine")
+_IDEATION_API_AGENT_SYSTEM_PROMPT = load_agentic_prompt_text("idea_api_ideation_agent")
+_INSIGHT_API_AGENT_SYSTEM_PROMPT = load_agentic_prompt_text("idea_api_insight_agent")
+
+
+class IdeationResponseAgentOutput(BaseModel):
+    response_message: str = Field(min_length=1)
+    response_kind: Literal["redirect", "needs_clarification", "ready_for_downstream", "ideation_contract_response"]
+    suggested_next_step: str = Field(min_length=1)
+    follow_up_questions: list[str] = Field(default_factory=list)
+    style_notes: list[str] = Field(default_factory=list)
+
+
+class InsightResponseAgentOutput(BaseModel):
+    response_message: str = Field(min_length=1)
+    response_kind: Literal["redirect", "needs_clarification", "ready_for_downstream", "ideation_contract_response"]
+    suggested_next_step: str = Field(min_length=1)
+    follow_up_questions: list[str] = Field(default_factory=list)
+    style_notes: list[str] = Field(default_factory=list)
+
+
+_IDEATION_MODEL = ANTHROPIC_MODEL_DEFAULT
+_MAX_PROMPT_CHARS = 120_000
+_MAX_TOOL_ROUNDS = 8
+_DEVIN2_PI_MODE_LABEL = "devin2_pi"
+
+
+def _response_mode_label() -> str:
+    return current_response_mode_label(env_value=os.environ.get("DEVFLOW_IDEATION_RESPONSE_MODE_LABEL"))
+
+
+def _use_devin2_pi_harness() -> bool:
+    return _response_mode_label() == _DEVIN2_PI_MODE_LABEL
+
+
+def _read_anthropic_key(*, repo_root: Path) -> str:
+    try:
+        return read_anthropic_api_key(repo_root=repo_root)
+    except RuntimeError as exc:
+        if "Anthropic API key not configured." not in str(exc):
+            raise
+        raise RuntimeError(
+            "Anthropic API key not configured for ideation arm. Expected runtime env, macOS keychain entry, or global ~/.devflow/keys/anthropic.json."
+        ) from exc
+
+
+def _ideation_api_settings(*, repo_root: Path) -> _AnthropicSettings:
+    return load_anthropic_api_settings(
+        repo_root=repo_root,
+        model=_IDEATION_MODEL,
+        max_tokens=1_200,
+    )
+
+
+def _tool_input_schema(spec: dict[str, Any]) -> dict[str, Any]:
+    properties: dict[str, Any] = {}
+    required: list[str] = []
+    for name, meta in dict(spec.get("parameters") or {}).items():
+        schema: dict[str, Any] = {"type": meta.get("type", "string")}
+        description = str(meta.get("description") or "").strip()
+        if description:
+            schema["description"] = description
+        enum_values = meta.get("enum")
+        if isinstance(enum_values, list) and enum_values:
+            schema["enum"] = list(enum_values)
+        properties[name] = schema
+        if meta.get("required") is True:
+            required.append(name)
+    return {
+        "type": "object",
+        "properties": properties,
+        "required": required,
+        "additionalProperties": False,
+    }
+
+
+def _repo_tools_from_specs(tool_specs: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [
+        {
+            "name": str(spec["name"]),
+            "description": str(spec.get("description") or ""),
+            "input_schema": _tool_input_schema(spec),
+        }
+        for spec in tool_specs
+    ]
+
+
+def _anthropic_messages_create(
+    *,
+    settings: _AnthropicSettings,
+    system_prompt: str,
+    messages: list[dict[str, Any]],
+    tools: list[dict[str, Any]],
+    timeout_seconds: int | None,
+) -> dict[str, Any]:
+    try:
+        return _shared_anthropic_messages_create(
+            settings=settings,
+            system_prompt=system_prompt,
+            messages=messages,
+            tools=tools,
+            timeout_seconds=timeout_seconds,
+        )
+    except RuntimeError as exc:
+        message = str(exc)
+        if not message.startswith("Anthropic API request failed:"):
+            raise
+        raise RuntimeError(message.replace("Anthropic API request failed:", "Anthropic ideation API request failed:", 1)) from exc
+
+
+def _assistant_text(response: dict[str, Any]) -> str:
+    return anthropic_response_text(response)
+
+
+def _tool_uses(response: dict[str, Any]) -> list[dict[str, Any]]:
+    return [dict(block) for block in list(response.get("content") or []) if block.get("type") == "tool_use"]
+
+
+def _run_api_agent(
+    *,
+    repo_root: Path,
+    prompt: dict[str, Any],
+    output_model: type[BaseModel],
+    settings: _AnthropicSettings,
+    tools: list[dict[str, Any]],
+    tool_dispatch: dict[str, Any],
+    system_prompt: str,
+    unsupported_tool_message: str,
+    no_json_message: str,
+    round_limit_message: str,
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    messages: list[dict[str, Any]] = [{"role": "user", "content": json.dumps(prompt, indent=2, sort_keys=True)}]
+    transcript: list[dict[str, Any]] = []
+
+    with activate_repo_tools(repo_root):
+        for _ in range(_MAX_TOOL_ROUNDS):
+            response = _anthropic_messages_create(
+                settings=settings,
+                system_prompt=system_prompt,
+                messages=messages,
+                tools=tools,
+                timeout_seconds=timeout_seconds,
+            )
+            transcript.append(response)
+            tool_calls = _tool_uses(response)
+            if not tool_calls:
+                raw_text = _assistant_text(response)
+                raw_json = _extract_json_hardened(raw_text)
+                if raw_json is None:
+                    raise RuntimeError(no_json_message)
+                model = output_model.model_validate_json(raw_json)
+                return model, AgentRunEnvelope(
+                    prompt=prompt,
+                    response=model.model_dump(),
+                    raw_stdout=json.dumps({"model": settings.model, "transcript": transcript}, indent=2, sort_keys=True),
+                    raw_stderr="",
+                )
+
+            messages.append({"role": "assistant", "content": list(response.get("content") or [])})
+            tool_results: list[dict[str, Any]] = []
+            for tool_call in tool_calls:
+                name = str(tool_call.get("name") or "").strip()
+                if name not in tool_dispatch:
+                    raise RuntimeError(unsupported_tool_message.format(name=name))
+                tool_fn = tool_dispatch[name]
+                tool_input = dict(tool_call.get("input") or {})
+                tool_output = tool_fn(**tool_input)
+                tool_results.append(
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": str(tool_call.get("id") or ""),
+                        "content": json.dumps(tool_output, indent=2, sort_keys=True),
+                    }
+                )
+            messages.append({"role": "user", "content": tool_results})
+
+    raise RuntimeError(round_limit_message)
+
+
+def _run_ideation_api_agent(
+    *,
+    repo_root: Path,
+    prompt: dict[str, Any],
+    output_model: type[BaseModel],
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    return _run_api_agent(
+        repo_root=repo_root,
+        prompt=prompt,
+        output_model=output_model,
+        settings=_ideation_api_settings(repo_root=repo_root),
+        tools=_repo_tools_from_specs(REPO_TOOL_SPECS),
+        tool_dispatch=REPO_TOOL_DISPATCH,
+        system_prompt=_IDEATION_API_AGENT_SYSTEM_PROMPT,
+        unsupported_tool_message="Ideation API agent requested unsupported tool: {name!r}",
+        no_json_message="Ideation API agent returned no JSON payload.",
+        round_limit_message="Ideation API agent exceeded tool-round limit without returning a final JSON response.",
+        timeout_seconds=timeout_seconds,
+    )
+
+
+def _run_insight_api_agent(
+    *,
+    repo_root: Path,
+    prompt: dict[str, Any],
+    output_model: type[BaseModel],
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    return _run_api_agent(
+        repo_root=repo_root,
+        prompt=prompt,
+        output_model=output_model,
+        settings=_ideation_api_settings(repo_root=repo_root),
+        tools=_repo_tools_from_specs(INSIGHT_REPO_TOOL_SPECS),
+        tool_dispatch=INSIGHT_REPO_TOOL_DISPATCH,
+        system_prompt=_INSIGHT_API_AGENT_SYSTEM_PROMPT,
+        unsupported_tool_message="Insight API agent requested unsupported tool: {name!r}",
+        no_json_message="Insight API agent returned no JSON payload.",
+        round_limit_message="Insight API agent exceeded tool-round limit without returning a final JSON response.",
+        timeout_seconds=timeout_seconds,
+    )
+
+
+
+def _request_response_llm_kwargs() -> dict[str, Any]:
+    cfg = _read_toml(_global_devflow_dir() / "config.toml")
+    llm_mode = str(cfg.get("llm_mode") or "").strip().lower()
+    if llm_mode == "api":
+        return {
+            "transport": "api",
+        }
+    return {}
+
+
+def _run_request_response_loop_call(
+    *,
+    repo_root: Path,
+    prompt: dict[str, Any],
+    output_model: type[BaseModel],
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    result = invoke_llm(
+        LlmInvocationRequest(
+            purpose=str(prompt.get("task") or "idea_devin_post_context_response"),
+            repo_root=repo_root,
+            prompt="",
+            prompt_payload=prompt,
+            delivery_model="final_only",
+            interaction_model="request_response",
+            response_contract="json_only",
+            timeout_seconds=timeout_seconds,
+            strength="light",
+            **_request_response_llm_kwargs(),
+        )
+    )
+    if not result.ok:
+        raise RuntimeError(result.stderr or result.stdout or "Devin response loop LLM call failed.")
+    if not result.contract_ok or result.parsed_json is None:
+        raise RuntimeError(result.contract_error or result.stdout or "Devin response loop returned invalid JSON.")
+    model = output_model.model_validate(result.parsed_json)
+    envelope = AgentRunEnvelope(
+        prompt=prompt,
+        response=model.model_dump(),
+        raw_stdout=json.dumps(
+            {
+                "transport": result.transport,
+                "provider": result.provider,
+                "model": result.model,
+                "delivery_model": result.delivery_model,
+                "interaction_model": result.interaction_model,
+                "response_contract": result.response_contract,
+                "stdout": result.stdout,
+            },
+            indent=2,
+            sort_keys=True,
+        ),
+        raw_stderr=result.stderr,
+    )
+    return model, envelope
+
+
+def _run_devin2_pi_loop_call(
+    *,
+    repo_root: Path,
+    route_arm: Literal["ideation", "insight"],
+    prompt: dict[str, Any],
+    output_model: type[BaseModel],
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    pi_timeout_seconds = max(int(timeout_seconds or 0), 150) if timeout_seconds is not None else 150
+    result = run_devin2_pi_agent(
+        repo_root=repo_root,
+        stage_name=str(prompt.get("task") or "idea_devin_post_context_response"),
+        route_arm=route_arm,
+        context_payload=dict(prompt.get("context") or {}),
+        operational_guidance=list(prompt.get("guidance") or prompt.get("instructions") or []),
+        output_model=output_model,
+        timeout_seconds=pi_timeout_seconds,
+    )
+    model = output_model.model_validate(result.response_model.model_dump())
+    desired_response_kind: str | None = None
+    for item in list(prompt.get("guidance") or prompt.get("instructions") or []):
+        text = str(item or "")
+        marker = "Return response_kind='"
+        if marker not in text:
+            continue
+        desired_response_kind = text.split(marker, 1)[1].split("'", 1)[0].strip() or None
+        if desired_response_kind:
+            break
+    if desired_response_kind and hasattr(model, "response_kind"):
+        actual_response_kind = str(getattr(model, "response_kind") or "")
+        if desired_response_kind == "ideation_contract_response" and actual_response_kind in {
+            "needs_clarification",
+            "ready_for_downstream",
+        }:
+            model = model.model_copy(update={"response_kind": desired_response_kind})
+    if hasattr(model, "follow_up_questions"):
+        questions = list(getattr(model, "follow_up_questions") or [])
+        response_kind = str(getattr(model, "response_kind") or "") if hasattr(model, "response_kind") else ""
+        if response_kind == "ready_for_downstream":
+            model = model.model_copy(update={"follow_up_questions": []})
+        elif len(questions) > 1:
+            model = model.model_copy(update={"follow_up_questions": questions[:1]})
+    envelope = AgentRunEnvelope(
+        prompt=result.prompt_payload,
+        response=model.model_dump(),
+        raw_stdout=json.dumps(
+            {
+                "transport": result.invocation.transport,
+                "provider": result.invocation.provider,
+                "model": result.invocation.model,
+                "delivery_model": result.invocation.delivery_model,
+                "interaction_model": result.invocation.interaction_model,
+                "response_contract": result.invocation.response_contract,
+                "stdout": result.invocation.stdout,
+            },
+            indent=2,
+            sort_keys=True,
+        ),
+        raw_stderr=result.invocation.stderr,
+    )
+    return model, envelope
+
+
+def run_ideation_response_agent_step(
+    *,
+    repo_root: Path,
+    context_payload: dict[str, Any],
+    guidance: list[str],
+    timeout_seconds: int | None = None,
+) -> tuple[IdeationResponseAgentOutput, AgentRunEnvelope]:
+    prompt = _build_prompt(
+        stage_name="idea_devin_post_context_response",
+        context_payload=context_payload,
+        guidance=IDEATION_RESPONSE_DOCTRINE + guidance,
+        output_model=IdeationResponseAgentOutput,
+    )
+    if len(json.dumps(prompt)) > _MAX_PROMPT_CHARS:
+        artifact_path = _persist_large_context_artifact(
+            repo_root=repo_root,
+            stage_name="idea_devin_post_context_response",
+            context_payload=context_payload,
+        )
+        prompt = _build_artifact_reference_prompt(
+            stage_name="idea_devin_post_context_response",
+            output_model=IdeationResponseAgentOutput,
+            guidance=IDEATION_RESPONSE_DOCTRINE + guidance,
+            artifact_path=artifact_path,
+        )
+    if _use_devin2_pi_harness():
+        return _run_devin2_pi_loop_call(
+            repo_root=repo_root,
+            route_arm="ideation",
+            prompt=prompt,
+            output_model=IdeationResponseAgentOutput,
+            timeout_seconds=timeout_seconds,
+        )
+    model, envelope = _run_request_response_loop_call(
+        repo_root=repo_root,
+        prompt=prompt,
+        output_model=IdeationResponseAgentOutput,
+        timeout_seconds=timeout_seconds,
+    )
+    return model, envelope
+
+
+def run_insight_response_agent_step(
+    *,
+    repo_root: Path,
+    context_payload: dict[str, Any],
+    guidance: list[str],
+    timeout_seconds: int | None = None,
+) -> tuple[InsightResponseAgentOutput, AgentRunEnvelope]:
+    prompt = _build_prompt(
+        stage_name="idea_devin_post_context_response",
+        context_payload=context_payload,
+        guidance=IDEATION_RESPONSE_DOCTRINE + guidance,
+        output_model=InsightResponseAgentOutput,
+    )
+    if len(json.dumps(prompt)) > _MAX_PROMPT_CHARS:
+        artifact_path = _persist_large_context_artifact(
+            repo_root=repo_root,
+            stage_name="idea_devin_post_context_response",
+            context_payload=context_payload,
+        )
+        prompt = _build_artifact_reference_prompt(
+            stage_name="idea_devin_post_context_response",
+            output_model=InsightResponseAgentOutput,
+            guidance=IDEATION_RESPONSE_DOCTRINE + guidance,
+            artifact_path=artifact_path,
+        )
+    return _run_devin2_pi_loop_call(
+        repo_root=repo_root,
+        route_arm="insight",
+        prompt=prompt,
+        output_model=InsightResponseAgentOutput,
+        timeout_seconds=timeout_seconds,
+    )
+
+
+
+def persist_agent_run(*, pipeline_root: Path, node_id: str, envelope: AgentRunEnvelope) -> Path:
+    path = pipeline_root / "agent_runs" / f"{node_id}.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(envelope.model_dump(), indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return path

devflow_engine/idea/analyze.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .paths import get_idea_paths
+
+
+@dataclass(frozen=True)
+class EvidenceItem:
+    anchor: str
+    kind: str | None = None
+
+
+def _iso_now() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def collect_evidence(repo_root: Path, *, max_files: int = 50) -> list[EvidenceItem]:
+    """Collect lightweight evidence anchors.
+
+    We deliberately avoid copying raw file contents into artifacts by default
+    to reduce secret leakage risk.
+    """
+
+    evidence: list[EvidenceItem] = []
+    repo_name = repo_root.name
+
+    count = 0
+    for p in sorted(repo_root.rglob("*")):
+        if count >= max_files:
+            break
+        # Exclude internal/system paths.
+        posix = p.as_posix()
+        if "/.git/" in posix or "/.devflow/" in posix or p.name.startswith(".venv"):
+            continue
+        if p.is_dir():
+            continue
+        rel = p.relative_to(repo_root).as_posix()
+        anchor = f"fs:{repo_name}/{rel}"
+        kind = None
+        if rel.endswith(".md"):
+            kind = "docs"
+        elif rel.endswith(".py"):
+            kind = "python"
+        evidence.append(EvidenceItem(anchor=anchor, kind=kind))
+        count += 1
+
+    # Ensure non-empty evidence by falling back to repo root marker.
+    if not evidence:
+        evidence.append(EvidenceItem(anchor=f"fs:{repo_name}/"))
+
+    return evidence
+
+
+def write_analysis_artifact(
+    *,
+    repo_root: Path,
+    idea_id: str,
+    analysis_id: str,
+    git_ref: str | None = None,
+) -> Path:
+    paths = get_idea_paths(repo_root, idea_id=idea_id)
+    artifact_dir = paths.analysis_root / analysis_id
+    artifact_dir.mkdir(parents=True, exist_ok=True)
+
+    evidence = collect_evidence(repo_root)
+    analysis = {
+        "analysis_id": analysis_id,
+        "idea_id": idea_id,
+        "repo_root": str(repo_root),
+        "git_ref": git_ref,
+        "created_at": _iso_now(),
+        "evidence": [{"anchor": e.anchor, "kind": e.kind} for e in evidence],
+    }
+
+    (artifact_dir / "analysis.json").write_text(json.dumps(analysis, indent=2, sort_keys=True), encoding="utf-8")
+
+    report_lines: list[str] = []
+    report_lines.append(f"# Analysis {analysis_id}\n")
+    report_lines.append("## Provenance\n")
+    report_lines.append(f"- idea_id: {idea_id}\n")
+    report_lines.append(f"- repo_root: {repo_root}\n")
+    report_lines.append("\n## Evidence\n")
+    for e in evidence[:25]:
+        report_lines.append(f"- anchor: {e.anchor}\n")
+    report_lines.append("\n## Redaction\n")
+    report_lines.append("Redaction enabled: stored artifacts avoid raw file contents by default.\n")
+
+    (artifact_dir / "report.md").write_text("".join(report_lines), encoding="utf-8")
+    return artifact_dir

devflow_engine/idea/devin_chat_dag.py

@@ -0,0 +1 @@
+from devin.dag import *  # noqa: F401,F403

devflow_engine/idea/diff.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from .drafts import dest_filename_for_draft, parse_locked_header
+from .paths import get_idea_paths
+
+
+@dataclass(frozen=True)
+class DiffEntry:
+    draft_story_id: str
+    draft_path: str
+    canonical_path: str | None
+    status: str  # added|modified|unchanged
+
+
+def _load_manifest(repo_root: Path, *, idea_id: str, draft_set_id: str) -> dict:
+    paths = get_idea_paths(repo_root, idea_id=idea_id)
+    p = paths.drafts_root / draft_set_id / "manifest.json"
+    return json.loads(p.read_text(encoding="utf-8"))
+
+
+def compute_diff(
+    *,
+    repo_root: Path,
+    idea_id: str,
+    draft_set_id: str,
+    canonical_root: Path,
+) -> list[DiffEntry]:
+    manifest = _load_manifest(repo_root, idea_id=idea_id, draft_set_id=draft_set_id)
+    draft_paths = manifest.get("draft_paths") or []
+
+    entries: list[DiffEntry] = []
+    for rel in draft_paths:
+        draft_path = repo_root / rel
+        text = draft_path.read_text(encoding="utf-8", errors="ignore")
+        header = parse_locked_header(text)
+        draft_story_id = header.get("draft_story_id") or draft_path.stem
+
+        dest_name = dest_filename_for_draft(draft_path, header=header)
+        canon_path = canonical_root / dest_name
+
+        if not canon_path.exists():
+            status = "added"
+            canon = None
+        else:
+            canon = str(canon_path)
+            canon_text = canon_path.read_text(encoding="utf-8", errors="ignore")
+            status = "unchanged" if canon_text == text else "modified"
+
+        entries.append(
+            DiffEntry(
+                draft_story_id=str(draft_story_id),
+                draft_path=str(draft_path),
+                canonical_path=canon,
+                status=status,
+            )
+        )
+
+    entries.sort(key=lambda e: (e.draft_story_id, e.draft_path))
+    return entries
+
+
+def render_diff_text(entries: list[DiffEntry]) -> str:
+    counts = {"added": 0, "modified": 0, "unchanged": 0}
+    for e in entries:
+        counts[e.status] = counts.get(e.status, 0) + 1
+
+    lines: list[str] = []
+    lines.append("Diff summary")
+    lines.append(f"added: {counts.get('added', 0)}")
+    lines.append(f"modified: {counts.get('modified', 0)}")
+    lines.append(f"unchanged: {counts.get('unchanged', 0)}")
+    lines.append("")
+
+    for e in entries:
+        lines.append(f"- {e.draft_story_id}: {e.status}")
+        if e.canonical_path:
+            lines.append(f"  canonical: {e.canonical_path}")
+        lines.append(f"  draft: {e.draft_path}")
+
+    return "\n".join(lines) + "\n"
+
+
+def render_diff_json(entries: list[DiffEntry]) -> str:
+    data = {
+        "entries": [
+            {
+                "draft_story_id": e.draft_story_id,
+                "draft_path": e.draft_path,
+                "canonical": e.canonical_path,
+                "status": e.status,
+            }
+            for e in entries
+        ]
+    }
+    return json.dumps(data, indent=2, sort_keys=True) + "\n"