devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/devflow_state.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from datetime import UTC, datetime
+from typing import Any
+from urllib.request import Request, urlopen
+
+
+def _keychain_get(service: str, account: str) -> str | None:
+    try:
+        proc = subprocess.run(
+            ["security", "find-generic-password", "-s", service, "-a", account, "-w"],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+    except Exception:
+        return None
+    if proc.returncode != 0:
+        return None
+    value = proc.stdout.strip()
+    return value or None
+
+
+def _resolve_supabase_rest_config() -> tuple[str, str] | None:
+    if os.environ.get("PYTEST_CURRENT_TEST"):
+        return None
+    url = (
+        os.environ.get("DEVFLOW_SUPABASE_URL")
+        or os.environ.get("SUPABASE_URL")
+        or _keychain_get("Supabase URL", "Clarity")
+    )
+    key = (
+        os.environ.get("DEVFLOW_SUPABASE_SERVICE_KEY")
+        or os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
+        or os.environ.get("SUPABASE_SERVICE_KEY")
+        or _keychain_get("Supabase Service Key", "Clarity")
+    )
+    if not url or not key:
+        return None
+    return url.rstrip("/"), key
+
+
+def _postgrest_request(*, method: str, url: str, key: str, body: Any | None = None, prefer: str | None = None) -> Any:
+    payload = None if body is None else json.dumps(body).encode("utf-8")
+    req = Request(url, data=payload, method=method)
+    req.add_header("apikey", key)
+    req.add_header("Authorization", f"Bearer {key}")
+    if body is not None:
+        req.add_header("Content-Type", "application/json")
+    if prefer:
+        req.add_header("Prefer", prefer)
+    with urlopen(req, timeout=30) as resp:
+        raw = resp.read().decode("utf-8")
+        return json.loads(raw) if raw else None
+
+
+def _is_uuid_like(value: str | None) -> bool:
+    if not value:
+        return False
+    import re
+
+    pattern = r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+    return bool(re.fullmatch(pattern, value.strip()))
+
+
+def _resolve_dfs_project_id(*, url: str, key: str, project_id: str, display_path: str | None) -> str:
+    value = str(project_id or '').strip()
+    if _is_uuid_like(value):
+        return value
+    display_value = str(display_path or '').strip()
+    idea_id = None
+    if display_value.startswith('idea:'):
+        candidate = display_value.split(':', 1)[1].strip()
+        if candidate.startswith('idea_'):
+            idea_id = candidate
+    elif display_value.startswith('story:'):
+        candidate = display_value.split(':', 1)[1].strip()
+        if candidate.startswith('STORY:idea:'):
+            parts = candidate.split(':')
+            if len(parts) >= 3 and parts[2].startswith('idea_'):
+                idea_id = parts[2]
+    if not idea_id:
+        return value
+    try:
+        lookup_url = f"{url}/rest/v1/devflow_project_ideas?select=project_id&idea_id=eq.{idea_id}&limit=1"
+        rows = _postgrest_request(method='GET', url=lookup_url, key=key)
+    except Exception:
+        return value
+    if isinstance(rows, list) and rows:
+        resolved = str((rows[0] or {}).get('project_id') or '').strip()
+        if resolved:
+            return resolved
+    return value
+
+
+def publish_devflow_state(
+    *,
+    project_id: str,
+    run_id: str | None,
+    current_state: str,
+    current_status: str,
+    run_summary: str,
+    error_message: str | None = None,
+    display: str = "project",
+    display_path: str | None = None,
+) -> None:
+    config = _resolve_supabase_rest_config()
+    if config is None:
+        return
+    url, key = config
+    resolved_project_id = _resolve_dfs_project_id(url=url, key=key, project_id=project_id, display_path=display_path)
+    row = {
+        "project_id": resolved_project_id,
+        "run_id": run_id,
+        "current_state": current_state,
+        "current_status": current_status,
+        "error_code": None,
+        "error_message": error_message,
+        "display": display,
+        "display_path": display_path,
+        "run_summary": run_summary,
+        "updated_at": datetime.now(UTC).isoformat(),
+    }
+    try:
+        _postgrest_request(
+            method="POST",
+            url=f"{url}/rest/v1/devflow_state?on_conflict=project_id",
+            key=key,
+            body=row,
+            prefer="resolution=merge-duplicates",
+        )
+    except Exception:
+        return
+
+
+def fetch_story_statuses_from_supabase(*, story_ids: list[str]) -> dict[str, str]:
+    """Return Supabase story statuses keyed by story_id for the requested story ids."""
+    config = _resolve_supabase_rest_config()
+    if config is None:
+        return {}
+    normalized = [str(story_id or "").strip() for story_id in story_ids]
+    wanted = sorted({story_id for story_id in normalized if story_id})
+    if not wanted:
+        return {}
+
+    url, key = config
+    rows_by_story_id: dict[str, str] = {}
+    try:
+        from urllib.parse import quote
+
+        batch_size = 100
+        for offset in range(0, len(wanted), batch_size):
+            batch = wanted[offset : offset + batch_size]
+            encoded_values = ",".join(quote(f'"{story_id}"', safe="") for story_id in batch)
+            rows = _postgrest_request(
+                method="GET",
+                url=(
+                    f"{url}/rest/v1/devflow_idea_stories?select=story_id,status"
+                    f"&story_id=in.({encoded_values})"
+                ),
+                key=key,
+            )
+            if not isinstance(rows, list):
+                continue
+            for row in rows:
+                if not isinstance(row, dict):
+                    continue
+                story_id = str(row.get("story_id") or "").strip()
+                status = str(row.get("status") or "").strip()
+                if story_id:
+                    rows_by_story_id[story_id] = status
+    except Exception:
+        return {}
+    return rows_by_story_id
+
+
+def update_story_status_in_supabase(*, story_id: str, status: str) -> None:
+    """Mirror story implementation status to Supabase devflow_idea_stories."""
+    config = _resolve_supabase_rest_config()
+    if config is None:
+        return
+    url, key = config
+    try:
+        from urllib.parse import quote
+
+        row = {
+            "status": status,
+            "updated_at": datetime.now(UTC).isoformat(),
+        }
+        _postgrest_request(
+            method="PATCH",
+            url=f"{url}/rest/v1/devflow_idea_stories?story_id=eq.{quote(story_id)}",
+            key=key,
+            body=row,
+        )
+    except Exception:
+        return  # non-fatal — local state is source of truth

devflow_engine/devin2/__init__.py

@@ -0,0 +1,9 @@
+"""Isolated Devin 2.0 agent-first prompt helpers and PI-backed execution seam."""
+
+from .pi_runner import (
+    Devin2PiRunnerError,
+    Devin2PiRunResult,
+    run_devin2_pi_agent,
+)
+
+__all__ = ["Devin2PiRunResult", "Devin2PiRunnerError", "run_devin2_pi_agent"]

devflow_engine/devin2/agent_definition.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from importlib.resources import files
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_DEVIN2_PROMPT_PATH = _REPO_ROOT / "docs" / "prompts" / "devin2-agent-prompt.md"
+_PROJECT_JOURNAL_RELATIVE_DIR = ".devflow/journal"
+
+
+class Devin2AgentPromptPayload(BaseModel):
+    agent_name: str
+    route_arm: Literal["ideation", "insight", "iterate", "neither"]
+    prompt_path: str
+    prompt_markdown: str
+    project_journal: dict[str, Any]
+    separation_contract: dict[str, Any]
+
+
+def devin2_agent_prompt_path() -> Path:
+    if _DEVIN2_PROMPT_PATH.exists():
+        return _DEVIN2_PROMPT_PATH
+    return Path(str(files("devflow_engine").joinpath("docs", "prompts", "devin2-agent-prompt.md")))
+
+
+def load_devin2_agent_prompt() -> str:
+    return devin2_agent_prompt_path().read_text(encoding="utf-8").strip()
+
+
+def build_devin2_agent_prompt_payload(
+    *,
+    repo_root: Path,
+    route_arm: Literal["ideation", "insight", "iterate"],
+) -> dict[str, Any]:
+    journal_root = repo_root / _PROJECT_JOURNAL_RELATIVE_DIR
+    return Devin2AgentPromptPayload(
+        agent_name="Devin 2.0",
+        route_arm=route_arm,
+        prompt_path=str(devin2_agent_prompt_path()),
+        prompt_markdown=load_devin2_agent_prompt(),
+        project_journal={
+            "relative_dir": _PROJECT_JOURNAL_RELATIVE_DIR,
+            "absolute_dir": str(journal_root),
+            "entry_pattern": ".devflow/journal/YYYY-MM-DD.md",
+            "write_rule": (
+                "Persist material assumptions, decisions, and continuity notes here "
+                "when they matter across turns or later execution."
+            ),
+        },
+        separation_contract={
+            "baseline_path": "docs/prompts/devin-agent-prompt.md",
+            "v2_path": "docs/prompts/devin2-agent-prompt.md",
+            "prompt_role": "Defines Devin 2.0's conversational stance and ownership split.",
+            "operational_guidance_role": (
+                "Supplies turn-specific workflow rules, route constraints, and execution caveats."
+            ),
+            "runtime_contract_role": "Defines the JSON output schema and return contract only.",
+        },
+    ).model_dump()
+
+
+def build_inline_agent_prompt_payload(
+    *,
+    prompt_text: str,
+    route_arm: Literal["ideation", "insight", "iterate", "neither"],
+) -> dict[str, Any]:
+    return {
+        "agent_name": "inline_response_agent",
+        "route_arm": route_arm,
+        "prompt_text": str(prompt_text).strip(),
+    }
+
+
+def build_response_prompt_payload(
+    *,
+    stage_name: str,
+    context_payload: dict[str, Any],
+    operational_guidance: list[str],
+    output_model: type[BaseModel],
+    agent_prompt: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "task": stage_name,
+        "agent_prompt": agent_prompt,
+        "operational_guidance": list(operational_guidance),
+        "context": context_payload,
+        "runtime_contract": {
+            "output_schema": output_model.model_json_schema(),
+            "return_format": "json_only",
+        },
+    }
+
+
+def build_artifact_reference_response_prompt_payload(
+    *,
+    stage_name: str,
+    operational_guidance: list[str],
+    output_model: type[BaseModel],
+    agent_prompt: dict[str, Any],
+    artifact_path: Path,
+) -> dict[str, Any]:
+    return {
+        "task": stage_name,
+        "agent_prompt": agent_prompt,
+        "operational_guidance": list(operational_guidance)
+        + [
+            "The full context was persisted to disk because the prompt exceeded size limits.",
+            f"Read context from artifact_path={artifact_path}",
+        ],
+        "context_artifact": {
+            "artifact_path": str(artifact_path),
+        },
+        "runtime_contract": {
+            "output_schema": output_model.model_json_schema(),
+            "return_format": "json_only",
+        },
+    }

devflow_engine/devin2/pi_runner.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ValidationError
+
+from ..idea.traditional_stories import _extract_json
+from ..llm.invoke import (
+    LlmInvocationRequest,
+    LlmInvocationResult,
+    invoke_llm,
+    resolve_api_tier_request_overrides,
+    resolve_canonical_light_tier_model,
+)
+from .agent_definition import (
+    build_artifact_reference_response_prompt_payload,
+    build_devin2_agent_prompt_payload,
+    build_response_prompt_payload,
+)
+
+_PI_PROVIDER = "pi"
+_PI_BASE_CMD = "pi"
+_PI_DELIVERY = "argument"
+_PI_DEVFLOW_TOOLS_EXT = str(Path.home() / ".pi/agent/extensions/devflow-tools.ts")
+_PI_BASE_CMD_WITH_DEVFLOW_TOOLS = f"pi -e {_PI_DEVFLOW_TOOLS_EXT}"
+_MAX_PROMPT_CHARS = 120_000
+_PI_MODEL_ALIASES: dict[tuple[str | None, str], str] = {
+    ("minimax", "MiniMax Token 2.7"): "MiniMax-M2.7",
+}
+
+
+@dataclass(frozen=True)
+class Devin2PiRunResult:
+    response_model: BaseModel
+    prompt_payload: dict[str, Any]
+    invocation: LlmInvocationResult
+
+
+class Devin2PiRunnerError(RuntimeError):
+    """Raised when the Devin 2.0 PI harness fails or returns invalid output."""
+
+
+def _persist_context_artifact(*, repo_root: Path, stage_name: str, context_payload: dict[str, Any]) -> Path:
+    artifact_root = repo_root / ".devflow" / "agent_debug" / "devin2_context_artifacts"
+    artifact_root.mkdir(parents=True, exist_ok=True)
+    safe_stage_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", stage_name).strip("._") or "stage"
+    artifact_path = artifact_root / f"{safe_stage_name}.json"
+    artifact_path.write_text(json.dumps(context_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return artifact_path
+
+
+def _extract_response_json(stdout: str) -> Any:
+    extracted = _extract_json(stdout)
+    if extracted is None:
+        raise Devin2PiRunnerError("Devin 2.0 PI harness returned no JSON payload.")
+    try:
+        return json.loads(extracted)
+    except json.JSONDecodeError as exc:
+        raise Devin2PiRunnerError(f"Devin 2.0 PI harness returned invalid JSON: {exc}") from exc
+
+
+def _build_devin2_prompt_payload(
+    *,
+    repo_root: Path,
+    route_arm: Literal["ideation", "insight", "iterate"],
+    stage_name: str,
+    context_payload: dict[str, Any],
+    operational_guidance: list[str],
+    output_model: type[BaseModel],
+) -> dict[str, Any]:
+    agent_prompt = build_devin2_agent_prompt_payload(repo_root=repo_root, route_arm=route_arm)
+    prompt_payload = build_response_prompt_payload(
+        stage_name=stage_name,
+        context_payload=context_payload,
+        operational_guidance=operational_guidance,
+        output_model=output_model,
+        agent_prompt=agent_prompt,
+    )
+    serialized = json.dumps(prompt_payload, indent=2, sort_keys=True)
+    if len(serialized) <= _MAX_PROMPT_CHARS:
+        return prompt_payload
+
+    artifact_path = _persist_context_artifact(
+        repo_root=repo_root,
+        stage_name=stage_name,
+        context_payload=context_payload,
+    )
+    return build_artifact_reference_response_prompt_payload(
+        stage_name=stage_name,
+        operational_guidance=operational_guidance,
+        output_model=output_model,
+        agent_prompt=agent_prompt,
+        artifact_path=artifact_path,
+    )
+
+
+
+
+def _normalize_pi_model_name(*, provider: str | None, model: str | None) -> str | None:
+    normalized = str(model or "").strip()
+    if not normalized:
+        return None
+    return _PI_MODEL_ALIASES.get((provider, normalized), normalized)
+
+
+def _resolve_pi_model_spec() -> str | None:
+    provider: str | None = None
+    model: str | None = None
+    try:
+        overrides = resolve_api_tier_request_overrides(tier="light")
+        provider = str(overrides.get("provider") or "").strip().lower() or None
+        model = str(overrides.get("model") or "").strip() or None
+    except Exception:
+        model = resolve_canonical_light_tier_model()
+    normalized_model = _normalize_pi_model_name(provider=provider, model=model)
+    if provider and normalized_model and "/" not in normalized_model:
+        return f"{provider}/{normalized_model}"
+    return normalized_model
+
+def run_devin2_pi_agent(
+    *,
+    repo_root: Path,
+    stage_name: str,
+    route_arm: Literal["ideation", "insight", "iterate"],
+    context_payload: dict[str, Any],
+    operational_guidance: list[str],
+    output_model: type[BaseModel],
+    timeout_seconds: int | None = None,
+) -> Devin2PiRunResult:
+    """Run the Devin 2.0 prompt payload through the PI CLI harness.
+
+    This seam hard-pins the execution transport to the PI binary itself while
+    explicitly passing the canonical light-tier model from the user-managed
+    Supabase row ``devflow_settings.settings.tiers.light`` (falling back only
+    to the synced global DevFlow view when Supabase is unavailable). That keeps
+    Devin 2.0 PI runs off repo-local / legacy local config resolution.
+    """
+    repo_root.mkdir(parents=True, exist_ok=True)
+    prompt_payload = _build_devin2_prompt_payload(
+        repo_root=repo_root,
+        route_arm=route_arm,
+        stage_name=stage_name,
+        context_payload=context_payload,
+        operational_guidance=operational_guidance,
+        output_model=output_model,
+    )
+    prompt_text = json.dumps(prompt_payload, indent=2, sort_keys=True)
+    canonical_light_model = _resolve_pi_model_spec()
+    # Load the Devin-specific extension, which now applies its own explicit
+    # allowlist via PI's setActiveTools(...) seam.
+    pi_base_cmd = _PI_BASE_CMD_WITH_DEVFLOW_TOOLS
+
+    _MAX_PI_ATTEMPTS = 3
+    last_error: Exception | None = None
+    invocation = None
+    for _attempt in range(_MAX_PI_ATTEMPTS):
+        invocation = invoke_llm(
+            LlmInvocationRequest(
+                purpose=stage_name,
+                strength="light",
+                repo_root=repo_root,
+                prompt=prompt_text,
+                prompt_payload=prompt_payload,
+                delivery_model="final_only",
+                interaction_model="agentic",
+                response_contract="json_only",
+                timeout_seconds=timeout_seconds,
+                base_cmd=pi_base_cmd,
+                delivery=_PI_DELIVERY,
+                provider=_PI_PROVIDER,
+                model=canonical_light_model,
+            )
+        )
+        if not invocation.ok:
+            last_error = Devin2PiRunnerError(invocation.stderr or invocation.stdout or "Devin 2.0 PI harness failed.")
+            continue
+        # Check for parseable JSON in the response
+        parsed_json = invocation.parsed_json
+        if parsed_json is None:
+            try:
+                parsed_json = _extract_response_json(invocation.stdout)
+            except Devin2PiRunnerError as exc:
+                last_error = exc
+                continue
+        break
+    else:
+        if last_error is not None:
+            raise last_error
+        raise Devin2PiRunnerError("Devin 2.0 PI harness failed after all retry attempts.")
+    assert invocation is not None
+    try:
+        response_model = output_model.model_validate(parsed_json)
+    except ValidationError as exc:
+        raise Devin2PiRunnerError(f"Devin 2.0 PI harness returned schema-invalid JSON: {exc}") from exc
+
+    return Devin2PiRunResult(
+        response_model=response_model,
+        prompt_payload=prompt_payload,
+        invocation=invocation,
+    )

devflow_engine/devin_orchestration.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from devin.dag_two_arm import run_devin_two_arm_dag as run_devin_chat_dag
+from .stores.execution_store import ExecutionStore
+
+DEVIN_QUEUE_NAME = "assistant.devin"
+DEVIN_CHAT_TASK_KIND = "devin.chat"
+
+
+def enqueue_devin_chat_task(
+    *,
+    store: ExecutionStore,
+    project_id: str | None,
+    idea_id: str,
+    repo_root: Path,
+    text: str | None,
+    source_path: Path | None,
+    max_stories: int,
+    planes: list[str],
+    source_run_id: str | None = None,
+    response_mode_label: str | None = None,
+) -> dict[str, Any]:
+    idempotency_key = f"{idea_id}:{','.join(sorted(planes))}:{max_stories}:{text or ''}:{str(source_path or '')}:{response_mode_label or ''}"
+    return store.enqueue_task(
+        project_id=project_id,
+        queue_name=DEVIN_QUEUE_NAME,
+        task_kind=DEVIN_CHAT_TASK_KIND,
+        idempotency_key=idempotency_key,
+        source_run_id=source_run_id,
+        input_payload={
+            "idea_id": idea_id,
+            "text": text,
+            "source_path": str(source_path) if source_path else None,
+            "max_stories": max_stories,
+            "planes": planes,
+            "response_mode_label": response_mode_label,
+        },
+        context={"client": "devin", "workflow": "devin_chat"},
+    )
+
+
+def handle_devin_chat_task(store: ExecutionStore, repo_root: Path, task: dict[str, Any]) -> dict[str, Any]:
+    payload = dict(task.get("input") or {})
+    task_id = str(task["task_id"])
+    store.record_task_step(task_id=task_id, step_name="devin_chat.run", status="started", payload=payload)
+    result = run_devin_chat_dag(
+        repo_root=repo_root,
+        store=store,
+        idea_id=str(payload.get("idea_id") or ""),
+        text=payload.get("text"),
+        source_path=Path(str(payload["source_path"])) if payload.get("source_path") else None,
+        max_stories=int(payload.get("max_stories") or 3),
+        planes=[str(item) for item in (payload.get("planes") or ["api"])],
+        response_mode_label=(None if payload.get("response_mode_label") is None else str(payload.get("response_mode_label"))),
+    )
+    response = {
+        "idea_id": str(payload.get("idea_id") or ""),
+        "run_id": result.run_id,
+        "exit_code": result.exit_code,
+        "pipeline_dir": str(result.pipeline_dir),
+        "message": result.message,
+        "outcome": result.outcome,
+    }
+    store.emit_task_message(task_id=task_id, message_kind="result", stream="assistant", payload=response)
+    store.record_task_step(task_id=task_id, step_name="devin_chat.run", status="finished", payload=response)
+    return response

devflow_engine/docs/prompts/anti-patterns.md

@@ -0,0 +1,42 @@
+# Prompt anti-patterns
+
+This is the canonical home for prompt anti-pattern notes in `devflow_engine`.
+
+Scope split:
+- prompt design guidance and anti-patterns live under `docs/prompts/`
+- runtime skill artifacts live under `src/devflow_engine/skills/...` as described in `docs/devflow-skill-registry-spec.md`
+
+Keep this list short and concrete. Add an entry when it changes prompt quality or agent correctness in a recurring way.
+
+## Context smuggling
+
+**Definition:** leaking author-side orchestration, internal jargon, or non-local workflow context into a node/agent prompt that does not actually have that local context.
+
+Common examples:
+- telling an agent to "use the output from Node 1" or "continue what Node 2 already established" when that material is not present in the prompt payload
+- referencing GenAI DAG internals, route names, or pipeline stage assumptions that are meaningful to the prompt author but not to the receiving agent
+- writing prompts that depend on hidden repo/process knowledge instead of the explicitly provided task inputs
+
+Why it is harmful:
+- creates fake coherence: the prompt sounds specific, but the agent cannot actually ground the references
+- encourages hallucinated continuity and invented dependencies
+- makes prompts brittle when nodes are reused, reordered, or evaluated in isolation
+- hides the real contract the node needs in order to work correctly
+
+Preferred correction:
+- restate the local task in terms of the inputs the node actually receives
+- pass required upstream artifacts explicitly instead of naming invisible pipeline steps
+- replace internal orchestration references with bounded, self-contained instructions
+- if an agent truly needs workflow metadata, include that metadata as explicit input, not implied context
+
+## Hidden approval boundary
+
+Stub: prompts that imply permission to proceed across a human approval gate without explicitly stating whether approval has been given.
+
+## State inflation
+
+Stub: prompts that collapse distinct states such as drafted, persisted, queued, started, and completed into one vague notion of "in progress".
+
+## Tool-shaped prompt leakage
+
+Stub: prompts that bake in harness/tool mechanics too early, instead of expressing the task and only layering tool-specific constraints where they are actually required.