devflow-engine 1.0.0__py3-none-any.whl

devin/nodes/shared/idea_compliance_advisor_eval/runner.py

@@ -0,0 +1,252 @@
+"""Eval harness for Idea Compliance Advisor.
+
+Runs the idea_compliance_check advisor through the PI subprocess (matching the
+Pi-Pi pattern in devflow-tools.ts) against scenario fixtures and scores output.
+
+Usage:
+    python3 -m devin.nodes.shared.idea_compliance_advisor_eval.runner
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parents[5]
+_AGENT_FILE = ".pi/agents/idea-compliance-advisor/idea-compliance-advisor.md"
+_DEFAULT_REPO_ROOT = "/Users/devflow/repos/Spicy-Server"
+_DEFAULT_PROJECT = "proj_75f63d30"
+_DEFAULT_MODEL = "minimax/MiniMax-M2.7"
+
+
+@dataclass
+class ScenarioResult:
+    scenario_id: str
+    passed: bool
+    score: int
+    max_score: int
+    output: str
+    details: dict[str, int]
+    elapsed: float
+
+
+def load_agent_prompt() -> str:
+    content = _PROJECT_ROOT.joinpath(_AGENT_FILE).read_text()
+    match = re.match(r"^---\n[\s\S]*?\n---\n([\s\S]*)$", content, re.M)
+    return match.group(1).strip() if match else content
+
+
+def run_compliance_check(
+    idea_text: str,
+    idea_id: str,
+    repo_root: str = _DEFAULT_REPO_ROOT,
+    project_id: str = _DEFAULT_PROJECT,
+    model: str = _DEFAULT_MODEL,
+    timeout: int = 180,
+) -> tuple[str, float]:
+    system_prompt = load_agent_prompt()
+    context_block = (
+        f"Context: repo_root={repo_root}, project_id={project_id}, idea_id={idea_id}\n\n"
+        f"Shaped Idea:\n{idea_text}\n\n"
+        f"Question: Is this idea compliant?"
+    )
+    args = [
+        "pi", "--mode", "text", "--no-session", "--no-extensions",
+        "--model", model, "--tools", "read,grep,find,ls", "--thinking", "off",
+        "--append-system-prompt", system_prompt + "\n\n" + context_block,
+        f"Validate idea compliance for: {idea_text}",
+    ]
+    start = time.time()
+    proc = subprocess.Popen(
+        args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE, text=True, cwd=str(Path.home()),
+    )
+    stdout, _ = proc.communicate(timeout=timeout)
+    elapsed = time.time() - start
+    return stdout.strip(), elapsed
+
+
+def _extract_json(text: str) -> dict | None:
+    match = re.search(r"```json\s*([\s\S]+?)\s*```", text)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except json.JSONDecodeError:
+            pass
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    return None
+
+
+def _extract_verdict(text: str) -> str | None:
+    parsed = _extract_json(text)
+    if parsed and "verdict" in parsed:
+        return parsed["verdict"]
+    match = re.search(r"Verdict[:\s]+([A-Z_]+)", text)
+    if match:
+        return match.group(1)
+    match = re.search(r"\*\*(COMPLIANT|NON_COMPLIANT|NEEDS_REVIEW)\*\*", text)
+    if match:
+        return match.group(1)
+    return None
+
+
+def score_scenario(output: str, scenario: dict) -> ScenarioResult:
+    score = 0
+    max_score = sum(scenario["scoring"].values())
+    details = {}
+    parsed = _extract_json(output)
+    expected_verdict = scenario["expected_verdict"]
+
+    verdict = (parsed["verdict"] if parsed and "verdict" in parsed else
+               _extract_verdict(output)) or "?"
+
+    # Verdict correct (always scored)
+    if verdict == expected_verdict:
+        details["verdict_correct"] = scenario["scoring"].get("verdict_correct", 0)
+        score += details["verdict_correct"]
+    else:
+        details["verdict_correct"] = 0
+
+    if not parsed:
+        return ScenarioResult(
+            scenario_id=scenario["id"],
+            passed=score == max_score,
+            score=score,
+            max_score=max_score,
+            output=output[:500],
+            details=details,
+            elapsed=0.0,
+        )
+
+    def gate_status(key: str) -> str:
+        val = parsed.get(key, {})
+        if isinstance(val, dict):
+            return val.get("status", "")
+        return ""
+
+    # --- COMPLIANT scoring: all gates pass ---
+    if expected_verdict == "COMPLIANT":
+        gate_checks = [
+            ("gate_1_architecture", "pass", "gate_1_architecture_pass"),
+            ("gate_2_methods", "pass", "gate_2_methods_pass"),
+            ("gate_3_story_dup", "pass", "gate_3_story_dup_pass"),
+            ("gate_4_idea_dup", "pass", "gate_4_idea_dup_pass"),
+            ("gate_5_naming", "pass", "gate_5_naming_pass"),
+        ]
+        for gate_key, expected_status, score_key in gate_checks:
+            actual = gate_status(gate_key)
+            if actual == expected_status:
+                details[score_key] = scenario["scoring"].get(score_key, 0)
+                score += details[score_key]
+            else:
+                details[score_key] = 0
+
+    # --- NON_COMPLIANT scoring: hard gate violation ---
+    elif expected_verdict == "NON_COMPLIANT":
+        if gate_status("gate_1_architecture") == "fail":
+            details["gate_1_architecture_fail"] = scenario["scoring"].get(
+                "gate_1_architecture_fail", 0)
+            score += details["gate_1_architecture_fail"]
+        if gate_status("gate_2_methods") == "fail":
+            details["gate_2_methods_fail_tier3"] = scenario["scoring"].get(
+                "gate_2_methods_fail_tier3", 0)
+            score += details["gate_2_methods_fail_tier3"]
+        if gate_status("gate_3_story_dup") == "fail":
+            details["gate_3_story_dup_fail"] = scenario["scoring"].get(
+                "gate_3_story_dup_fail", 0)
+            score += details["gate_3_story_dup_fail"]
+        if gate_status("gate_4_idea_dup") == "fail":
+            details["gate_4_idea_dup_fail"] = scenario["scoring"].get(
+                "gate_4_idea_dup_fail", 0)
+            score += details["gate_4_idea_dup_fail"]
+
+    # --- NEEDS_REVIEW scoring ---
+    elif expected_verdict == "NEEDS_REVIEW":
+        if gate_status("gate_2_methods") == "needs_review":
+            details["gate_2_needs_review_tier2"] = scenario["scoring"].get(
+                "gate_2_needs_review_tier2", 0)
+            score += details["gate_2_needs_review_tier2"]
+        if gate_status("gate_4_idea_dup") == "needs_review":
+            details["gate_4_needs_review"] = scenario["scoring"].get(
+                "gate_4_needs_review", 0)
+            score += details["gate_4_needs_review"]
+        if gate_status("gate_5_naming") == "needs_review":
+            details["gate_5_needs_review"] = scenario["scoring"].get(
+                "gate_5_needs_review", 0)
+            score += details["gate_5_needs_review"]
+
+    # Issues checks (any issues present scores)
+    issues = parsed.get("issues", [])
+    issues_checks = [
+        "issues_specific",
+        "issues_describes_package_needed",
+        "issues_describes_overlap",
+        "issues_naming_noted",
+    ]
+    for k in issues_checks:
+        if k in scenario["scoring"] and any(issues):
+            details[k] = scenario["scoring"].get(k, 0)
+            score += details[k]
+            break
+
+    return ScenarioResult(
+        scenario_id=scenario["id"],
+        passed=score == max_score,
+        score=score,
+        max_score=max_score,
+        output=output[:500],
+        details=details,
+        elapsed=0.0,
+    )
+
+
+def run_all_scenarios(scenarios_path: str | Path | None = None) -> list[ScenarioResult]:
+    if scenarios_path is None:
+        scenarios_path = Path(__file__).parent / "scenarios.json"
+    scenarios = json.loads(Path(scenarios_path).read_text())
+    results = []
+    for scenario in scenarios:
+        print(f"\nRunning: {scenario['id']}")
+        output, elapsed = run_compliance_check(
+            idea_text=scenario["idea_text"],
+            idea_id=scenario["idea_id"],
+            repo_root=scenario["repo_root"],
+            project_id=scenario["project_id"],
+        )
+        result = score_scenario(output, scenario)
+        result.elapsed = elapsed
+        results.append(result)
+        verdict = _extract_verdict(output) or "?"
+        print(f"  Verdict: {verdict} | Score: {result.score}/{result.max_score} ({elapsed:.1f}s)")
+        if not result.passed:
+            print(f"  Missing: {[k for k, v in result.details.items() if v == 0]}")
+    return results
+
+
+def print_report(results: list[ScenarioResult]) -> None:
+    total = sum(r.score for r in results)
+    max_total = sum(r.max_score for r in results)
+    print(f"\n{'='*60}")
+    print(f"IDEA COMPLIANCE ADVISOR EVAL REPORT")
+    print(f"{'='*60}")
+    for r in results:
+        status = "✓ PASS" if r.passed else "✗ FAIL"
+        print(f"\n{status} {r.scenario_id}  {r.score}/{r.max_score}")
+        for key, val in r.details.items():
+            icon = "✓" if val > 0 else "✗"
+            print(f"    {icon} {key}: {val}")
+    print(f"\n{'='*60}")
+    print(f"TOTAL: {total}/{max_total}  ({100*total/max_total:.0f}%)")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    results = run_all_scenarios()
+    print_report(results)

devin/nodes/shared/idea_compliance_advisor_eval/scenarios.json

@@ -0,0 +1,75 @@
+[
+  {
+    "id": "compliant_memory_feature",
+    "description": "Memory summarization fits companion chat domain — uses existing pydantic-ai episodic pattern, no new package — all gates pass",
+    "idea_text": "Add a memory summarization feature that periodically condenses older conversation turns in the companion chat to save context window space while preserving key facts",
+    "idea_id": "proj_75f63d30_idea_eval_c001",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "expected_verdict": "COMPLIANT",
+    "scoring": {
+      "verdict_correct": 2,
+      "gate_1_architecture_pass": 1,
+      "gate_2_methods_pass": 1,
+      "gate_3_story_dup_pass": 1,
+      "gate_4_idea_dup_pass": 1,
+      "gate_5_naming_pass": 1
+    }
+  },
+  {
+    "id": "non_compliant_billing_portal",
+    "description": "Invoice/portal system is fundamental architecture violation for companion chat — NON_COMPLIANT on gate 1",
+    "idea_text": "Add an invoice management system with PDF generation, payment tracking, and overdue reminder emails for clients",
+    "idea_id": "proj_75f63d30_idea_eval_nc001",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "expected_verdict": "NON_COMPLIANT",
+    "scoring": {
+      "verdict_correct": 2,
+      "gate_1_architecture_fail": 1,
+      "issues_specific": 1
+    }
+  },
+  {
+    "id": "needs_review_new_pkg_sentiment",
+    "description": "Sentiment analysis is a new domain for companion chat, not replicating existing patterns — NEEDS_REVIEW Tier 2 on gate 2",
+    "idea_text": "Add a new gating context type that evaluates user sentiment before allowing companion responses above a certain emotional intensity threshold",
+    "idea_id": "proj_75f63d30_idea_eval_c002",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "expected_verdict": "NEEDS_REVIEW",
+    "scoring": {
+      "verdict_correct": 2,
+      "gate_2_needs_review_tier2": 1,
+      "issues_describes_package_needed": 1
+    }
+  },
+  {
+    "id": "non_compliant_stripe_duplicates_existing",
+    "description": "Spicy-Server already has payment infrastructure — adding Stripe replicates existing pattern — NON_COMPLIANT Tier 3",
+    "idea_text": "Add Stripe payment processing so users can upgrade their subscription tier directly from the app",
+    "idea_id": "proj_75f63d30_idea_eval_nc002",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "expected_verdict": "NON_COMPLIANT",
+    "scoring": {
+      "verdict_correct": 2,
+      "gate_2_methods_fail_tier3": 1,
+      "issues_specific": 1
+    }
+  },
+  {
+    "id": "needs_review_naming_collision",
+    "description": "Idea has a naming collision that needs resolution — NEEDS_REVIEW on gate 5",
+    "idea_text": "Add a module called SentimentAnalyzer that processes user messages and returns emotional intensity scores",
+    "idea_id": "proj_75f63d30_idea_eval_nr002",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "expected_verdict": "NEEDS_REVIEW",
+    "scoring": {
+      "verdict_correct": 2,
+      "gate_5_needs_review": 1,
+      "issues_naming_noted": 1
+    }
+  }
+]

devin/nodes/shared/models.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+RouteArm = Literal['ideation', 'insight', 'iterate', 'neither']
+
+class DevinChatDagEvent(BaseModel):
+    repo_root: str
+    idea_id: str
+    raw_text: str | None = None
+    source_path: str | None = None
+    max_stories: int = 0
+    planes: list[str] = Field(default_factory=list)
+    pipeline_key: str
+    source_session_id: str | None = None  # original session_id from the incoming message row
+
+class DevinAgentResponse(BaseModel):
+    response_message: str = Field(min_length=1)
+    response_kind: Literal[
+        'redirect',
+        'needs_clarification',
+        'ready_for_downstream',
+        'ideation_contract_response',
+        'insight_response',
+        'operational_alert',
+        'needs_context',
+        'completed',
+        'blocked',
+        'needs_more_context',
+        'promote_to_idea',
+        'route_to_insight',
+    ]
+    suggested_next_step: str = Field(min_length=1)
+    follow_up_questions: list[str] = Field(default_factory=list)
+    style_notes: list[str] = Field(default_factory=list)
+
+class ScenarioResult(BaseModel):
+    scenario_name: str
+    passed: bool
+    actual_output: dict[str, Any]
+    expected_behavior: dict[str, Any]
+    notes: list[str] = Field(default_factory=list)

devin/nodes/shared/post.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+from devflow_engine.vendor.datalumina_genai.core.nodes.base import Node
+from devflow_engine.vendor.datalumina_genai.core.task import TaskContext
+from .helpers import build_human_facing_message, dfs_terminal, maybe_post_agent_message, pipeline_root, resolve_project_id, store_run, write_json
+
+class DevinResponsePostNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        event = task_context.event
+        repo_root = Path(event.repo_root)
+        store, run_id = store_run()
+        node_exec_id = store.create_node_attempt(run_id=run_id, node_id='devin_response_post', node_name='DevinResponsePost', attempt=1)
+        raw_project_id = str(task_context.metadata.get('project_id') or resolve_project_id(repo_root, idea_id=event.idea_id))
+        # Use None for project_id if it's not a valid UUID (i.e., unregistered)
+        try:
+            uuid.UUID(raw_project_id)
+            project_id = raw_project_id
+        except Exception:
+            project_id = None
+        response_payload = dict(task_context.metadata.get('response_guidance') or {})
+        if not response_payload:
+            raise RuntimeError('missing response payload for DevinResponsePostNode')
+        session_id = event.source_session_id or f'idea:{raw_project_id}:{event.idea_id}'
+        row = {'session_id': session_id, 'from_agent': 'devin', 'to_agent': 'charlie', 'message': str(response_payload.get('response_message') or '').strip(), 'metadata': {'project_id': raw_project_id, 'idea_id': event.idea_id, 'pipeline_key': event.pipeline_key, 'response_kind': response_payload.get('response_kind'), 'suggested_next_step': response_payload.get('suggested_next_step'), 'follow_up_questions': list(response_payload.get('follow_up_questions') or [])}, 'project_id': project_id, 'message_type': 'reply'}
+        created = maybe_post_agent_message(row=row)
+        post_path = pipeline_root(repo_root, idea_id=event.idea_id, pipeline_key=event.pipeline_key) / 'devin_response_post.json'
+        post_payload = {'request': row, 'response_record': created}
+        write_json(post_path, post_payload)
+        store.add_artifact(run_id=run_id, node_exec_id=node_exec_id, kind='idea_devin_response_post', uri=str(post_path), metadata={'session_id': session_id, 'response_kind': response_payload.get('response_kind')})
+        store.mark_node_finished(node_exec_id=node_exec_id, status='succeeded', output={'session_id': session_id, 'response_kind': response_payload.get('response_kind')})
+        task_context.metadata['response_post'] = post_payload
+        task_context.metadata['outcome'] = {'status': response_payload.get('response_kind') or 'completed', 'response_post': post_payload, **response_payload}
+        task_context.metadata['message'] = build_human_facing_message(response_payload=response_payload)
+        task_context.metadata['exit_code'] = 0
+        dfs_terminal(project_id=project_id, run_id=run_id, summary='Devin response completed', current_state='completed', current_status='completed', idea_id=event.idea_id)
+        task_context.stop_workflow()
+        return task_context

devin/nodes/shared/router.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from devflow_engine.llm.invoke import LlmInvocationRequest, invoke_llm
+
+
+def llm_route(*, raw_text: str, repo_root: Path, project_id: str | None = None) -> tuple[str, dict[str, Any]]:
+    """Route a user message to insight, ideation, or neither using a lightweight LLM call.
+    
+    Falls back to deterministic classification if the LLM call fails or times out.
+    """
+    prompt = f"""You are a router for a Devin two-arm agent system. Read the user's message and decide which arm should handle it.
+
+ARM DEFINITIONS:
+- insight: questions about the current state of the codebase, repo, project, queues, workers, architecture, runtime behavior, or "can you see/find/verify" requests. Questions that need file-reading or system inspection.
+- iterate: bounded implementation requests on an existing surface, like fixing a concrete bug, making a small behavior change, or tightening a specific UI on an existing page or workflow.
+- ideation: forward-looking requests about building new features, adding/changing/creating software, planning implementations, shaping product requirements. Requests that shape or plan software changes.
+- neither: anything that is not about software delivery or DevFlow operations (e.g., casual conversation, non-software topics).
+
+USER MESSAGE:
+{raw_text}
+
+Respond with ONLY a JSON object:
+{{"route_arm": "insight" | "iterate" | "ideation" | "neither", "reason": "one sentence explanation"}}"""  # noqa: E501
+
+    prompt_payload = {
+        "raw_text": raw_text,
+        "repo_root": str(repo_root),
+        "project_id": project_id,
+    }
+
+    try:
+        result = invoke_llm(
+            LlmInvocationRequest(
+                purpose="devin_route_arm",
+                repo_root=repo_root,
+                prompt=prompt,
+                prompt_payload=prompt_payload,
+                delivery_model="final_only",
+                interaction_model="request_response",
+                response_contract="json_only",
+                timeout_seconds=15,
+                strength="light",
+            )
+        )
+        if result.ok and result.parsed_json:
+            arm = str(result.parsed_json.get("route_arm") or "neither").strip().lower()
+            if arm not in ("insight", "iterate", "ideation", "neither"):
+                arm = "neither"
+            reason = str(result.parsed_json.get("reason") or "")
+            return arm, {
+                "route_arm": arm,
+                "reason": reason,
+                "classifier": "llm_router_v1",
+            }
+    except Exception:
+        pass
+
+    # Fallback to deterministic
+    return _deterministic_fallback(raw_text)
+
+
+def _deterministic_fallback(raw_text: str) -> tuple[str, dict[str, Any]]:
+    """Fast deterministic fallback when LLM routing is unavailable."""
+    lowered = raw_text.lower()
+    insight_signals = {"status", "queue", "worker", "repo", "code", "logs", "error",
+                       "why", "how", "where", "what", "investigate", "explain",
+                       "architecture", "runtime", "process", "pipeline", "see",
+                       "verify", "find", "search", "look", "check", "locate"}
+    iterate_signals = {"fix", "broken", "breaks", "bug", "change", "tweak", "adjust",
+                       "update", "existing", "current", "button", "form", "modal",
+                       "page", "route", "screen", "component", "endpoint", "workflow"}
+    ideation_signals = {"app", "application", "api", "automation", "backend", "bug",
+                        "dashboard", "feature", "frontend", "integration", "mobile",
+                        "portal", "saas", "software", "system", "tool", "ui",
+                        "web", "website", "workflow"}
+    non_software = {"blog post", "flyer", "hiring", "logo", "marketing",
+                    "newsletter", "podcast", "seo", "social media", "video script"}
+
+    insight_hits = [t for t in insight_signals if t in lowered]
+    iterate_hits = [t for t in iterate_signals if t in lowered]
+    ideation_hits = [t for t in ideation_signals if t in lowered]
+    non_hits = [t for t in non_software if t in lowered]
+
+    # Explicit phrase checks
+    verification_phrases = (
+        "can you see", "can you find", "can you locate", "please verify",
+        "can you check", "do you see", "do you have the file", "are the files in",
+    )
+    if any(phrase in lowered for phrase in verification_phrases):
+        return "insight", {"route_arm": "insight", "reason": "repo content verification request", "classifier": "deterministic_fallback"}
+    if any(t in lowered for t in ("queue", "worker", "status", "repo", "codebase",
+                                   "how does", "where is", "why is", "what is happening")):
+        return "insight", {"route_arm": "insight", "reason": "operational/status question", "classifier": "deterministic_fallback"}
+    if any(t in lowered for t in ("fix", "bug fix", "small change", "quick change", "tweak", "adjust", "update")) and any(t in lowered for t in ("existing", "current", "page", "route", "component", "button", "form", "modal", "screen", "endpoint", "workflow", "bug", "error", "broken")):
+        return "iterate", {"route_arm": "iterate", "reason": "bounded change request on an existing surface", "classifier": "deterministic_fallback"}
+    if iterate_hits and not insight_hits and len(iterate_hits) >= len(ideation_hits):
+        return "iterate", {"route_arm": "iterate", "reason": "targeted implementation request", "classifier": "deterministic_fallback"}
+    if ideation_hits and len(ideation_hits) >= len(non_hits):
+        return "ideation", {"route_arm": "ideation", "reason": "software/product request", "classifier": "deterministic_fallback"}
+    if any(t in lowered for t in ("build", "create", "implement", "ship", "plan")) and not non_hits and not insight_hits:
+        return "ideation", {"route_arm": "ideation", "reason": "forward-looking implementation request", "classifier": "deterministic_fallback"}
+    if insight_hits and "project" in lowered:
+        return "insight", {"route_arm": "insight", "reason": "project-specific investigation", "classifier": "deterministic_fallback"}
+    return "neither", {"route_arm": "neither", "reason": "not clearly software delivery work", "classifier": "deterministic_fallback"}