devflow-engine 1.0.0__py3-none-any.whl

devin/nodes/shared/filemaker_expert_eval/scenarios.json

@@ -0,0 +1,65 @@
+[
+  {
+    "id": "layout_analysis",
+    "description": "Expert should identify all layouts in the 'My Page . Cleaner' section with counts and purposes",
+    "question": "What layouts does the 'My Page . Cleaner' section consist of? List all layouts in that section.",
+    "expected_keys": ["My Page . Cleaner", "My Page . Cleaner . Availability", "My Page . Cleaner . iPhone", "KanbanDetails", "geoLocation"],
+    "expected_structure": "table with layout name, ID, objects, purpose",
+    "scoring": {
+      "all_layouts_found": 2,
+      "object_counts": 1,
+      "purpose_descriptions": 1,
+      "table_format": 1
+    }
+  },
+  {
+    "id": "script_flow",
+    "description": "Expert should identify key scripts in the inspection completion workflow and explain what happens",
+    "question": "What scripts handle the inspection completion workflow? What happens when an inspector marks an area as complete?",
+    "expected_keys": ["inspections . layout . save", "inspections . launch", "loadInspectionData", "loadAreas", "complete"],
+    "expected_structure": "list of scripts with IDs and step counts",
+    "scoring": {
+      "key_scripts_identified": 2,
+      "script_ids_present": 1,
+      "step_counts_present": 1,
+      "completion_logic_described": 1
+    }
+  },
+  {
+    "id": "data_model",
+    "description": "Expert should map the relationships between Inspector, Worksheet, and WorkOrders tables",
+    "question": "What are the key relationships between Inspector, Worksheet, and WorkOrders tables? Show the key foreign key fields.",
+    "expected_keys": ["_worksheetID", "_custID", "_cleanerID", "Inspector", "Worksheet", "WorkOrders", "foreign key"],
+    "expected_structure": "table with direction, field, purpose columns",
+    "scoring": {
+      "all_three_tables_mentioned": 2,
+      "foreign_key_fields_identified": 2,
+      "relationship_directions_clear": 1
+    }
+  },
+  {
+    "id": "journey_mapping",
+    "description": "Expert should trace a customer journey through layouts and scripts",
+    "question": "Map the customer journey for creating and completing an inspection. What layouts, scripts, and buttons are involved from start to finish?",
+    "expected_keys": ["Account Page", "Inspection Form III", "inspections . launch", "My Page", "save"],
+    "expected_structure": "phases with entry point, scripts, layouts",
+    "scoring": {
+      "entry_point_identified": 1,
+      "multiple_phases": 2,
+      "layouts_and_scripts_per_phase": 2,
+      "completion_end_state": 1
+    }
+  },
+  {
+    "id": "cruft_detection",
+    "description": "Expert should identify code health issues from cruft summary",
+    "question": "What code health issues (uncalled scripts, unreachable layouts, unused fields) does Clean Sweep have?",
+    "expected_keys": ["uncalled_scripts", "unreachable_layouts", "count", "0"],
+    "expected_structure": "per-category counts with item lists",
+    "scoring": {
+      "all_categories_covered": 2,
+      "counts_present": 1,
+      "specific_items_if_any": 1
+    }
+  }
+]

devin/nodes/shared/goldilocks_advisor_eval/runner.py

@@ -0,0 +1,214 @@
+"""Eval harness for Goldilocks Advisor.
+
+Runs the goldilocks_check advisor through the PI subprocess (matching the
+Pi-Pi pattern in devflow-tools.ts) against scenario fixtures and scores output.
+
+Usage:
+    python3 -m devin.nodes.shared.goldilocks_advisor_eval.runner
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parents[5]
+_AGENT_FILE = ".pi/agents/goldilocks-advisor/goldilocks-advisor.md"
+_DEFAULT_REPO_ROOT = "/Users/devflow/repos/Spicy-Server"
+_DEFAULT_PROJECT = "proj_75f63d30"
+_DEFAULT_MODEL = "minimax/MiniMax-M2.7"
+
+
+@dataclass
+class ScenarioResult:
+    scenario_id: str
+    passed: bool
+    score: int
+    max_score: int
+    output: str
+    details: dict[str, int]
+    elapsed: float
+
+
+def load_agent_prompt() -> str:
+    content = _PROJECT_ROOT.joinpath(_AGENT_FILE).read_text()
+    match = re.match(r"^---\n[\s\S]*?\n---\n([\s\S]*)$", content, re.M)
+    return match.group(1).strip() if match else content
+
+
+def run_goldilocks_check(
+    idea_text: str,
+    idea_id: str,
+    repo_root: str = _DEFAULT_REPO_ROOT,
+    project_id: str = _DEFAULT_PROJECT,
+    model: str = _DEFAULT_MODEL,
+    timeout: int = 90,
+) -> tuple[str, float]:
+    system_prompt = load_agent_prompt()
+    context_block = (
+        f"Context: repo_root={repo_root}, project_id={project_id}, idea_id={idea_id}\n\n"
+        f"Shaped Idea:\n{idea_text}\n\n"
+        f"Question: Estimate story count and evaluate scoping"
+    )
+    args = [
+        "pi", "--mode", "text", "--no-session", "--no-extensions",
+        "--model", model, "--tools", "read,grep,find,ls", "--thinking", "off",
+        "--append-system-prompt", system_prompt + "\n\n" + context_block,
+        f"Goldilocks scoping check for: {idea_text}",
+    ]
+    start = time.time()
+    proc = subprocess.Popen(
+        args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE, text=True, cwd=str(Path.home()),
+    )
+    stdout, _ = proc.communicate(timeout=timeout)
+    elapsed = time.time() - start
+    return stdout.strip(), elapsed
+
+
+def _extract_json(text: str) -> dict | None:
+    # Try to extract JSON from markdown code block or raw text
+    match = re.search(r"```json\s*([\s\S]+?)\s*```", text)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except json.JSONDecodeError:
+            pass
+    # Try raw JSON
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    return None
+
+
+def score_scenario(output: str, scenario: dict) -> ScenarioResult:
+    score = 0
+    max_score = sum(scenario["scoring"].values())
+    details = {}
+    parsed = _extract_json(output)
+    expected_verdict = scenario["expected_verdict"]
+
+    # Verdict check
+    verdict = None
+    if parsed and "verdict" in parsed:
+        verdict = parsed["verdict"]
+    if verdict == expected_verdict:
+        details["verdict_correct"] = scenario["scoring"].get("verdict_correct", 0)
+        score += details["verdict_correct"]
+    else:
+        details["verdict_correct"] = 0
+
+    # Story estimate present
+    if parsed and "story_estimate" in parsed:
+        details["story_estimate_present"] = scenario["scoring"].get("story_estimate_present", 0)
+        score += details["story_estimate_present"]
+    else:
+        details["story_estimate_present"] = 0
+
+    # Story estimate in range for just_right
+    if expected_verdict == "just_right" and parsed:
+        est = parsed.get("story_estimate", 0)
+        if 2 <= est <= 20:
+            details["story_estimate_in_range"] = scenario["scoring"].get("story_estimate_in_range", 0)
+            score += details["story_estimate_in_range"]
+        else:
+            details["story_estimate_in_range"] = 0
+    elif expected_verdict == "split" and parsed:
+        est = parsed.get("story_estimate", 0)
+        if est > 20:
+            details["story_estimate_over_20"] = scenario["scoring"].get("story_estimate_over_20", 0)
+            score += details["story_estimate_over_20"]
+        else:
+            details["story_estimate_over_20"] = 0
+    elif expected_verdict == "enrichment_needed" and parsed:
+        gaps = parsed.get("enrichment_gaps", [])
+        if gaps:
+            details["enrichment_gaps_filled"] = scenario["scoring"].get("enrichment_gaps_filled", 0)
+            score += details["enrichment_gaps_filled"]
+        else:
+            details["enrichment_gaps_filled"] = 0
+    else:
+        details["story_estimate_in_range"] = 0
+
+    # Split suggestions
+    if expected_verdict == "split":
+        splits = parsed.get("split_suggestions", []) if parsed else []
+        if splits:
+            details["split_suggestions_present"] = scenario["scoring"].get("split_suggestions_present", 0)
+            score += details["split_suggestions_present"]
+        else:
+            details["split_suggestions_present"] = 0
+
+    # Cohesive area present
+    if parsed and "cohesive_area" in parsed:
+        details["cohesive_area_present"] = scenario["scoring"].get("cohesive_area_present", 0)
+        score += details["cohesive_area_present"]
+    else:
+        details["cohesive_area_present"] = 0
+
+    # Reasoning present
+    if parsed and "reasoning" in parsed:
+        details["reasoning_present"] = scenario["scoring"].get("reasoning_present", 0)
+        score += details["reasoning_present"]
+    else:
+        details["reasoning_present"] = 0
+
+    return ScenarioResult(
+        scenario_id=scenario["id"],
+        passed=score == max_score,
+        score=score,
+        max_score=max_score,
+        output=output[:500],
+        details=details,
+        elapsed=0.0,
+    )
+
+
+def run_all_scenarios(scenarios_path: str | Path | None = None) -> list[ScenarioResult]:
+    if scenarios_path is None:
+        scenarios_path = Path(__file__).parent / "scenarios.json"
+    scenarios = json.loads(Path(scenarios_path).read_text())
+    results = []
+    for scenario in scenarios:
+        print(f"\nRunning: {scenario['id']}")
+        output, elapsed = run_goldilocks_check(
+            idea_text=scenario["idea_text"],
+            idea_id=scenario["idea_id"],
+            repo_root=scenario["repo_root"],
+            project_id=scenario["project_id"],
+        )
+        result = score_scenario(output, scenario)
+        result.elapsed = elapsed
+        results.append(result)
+        verdict = _extract_json(output).get("verdict", "?") if _extract_json(output) else "?"
+        print(f"  Verdict: {verdict} | Score: {result.score}/{result.max_score} ({elapsed:.1f}s)")
+        if not result.passed:
+            print(f"  Missing: {[k for k, v in result.details.items() if v == 0]}")
+    return results
+
+
+def print_report(results: list[ScenarioResult]) -> None:
+    total = sum(r.score for r in results)
+    max_total = sum(r.max_score for r in results)
+    print(f"\n{'='*60}")
+    print(f"GOLDILOCKS ADVISOR EVAL REPORT")
+    print(f"{'='*60}")
+    for r in results:
+        status = "✓ PASS" if r.passed else "✗ FAIL"
+        print(f"\n{status} {r.scenario_id}  {r.score}/{r.max_score}")
+        for key, val in r.details.items():
+            icon = "✓" if val > 0 else "✗"
+            print(f"    {icon} {key}: {val}")
+    print(f"\n{'='*60}")
+    print(f"TOTAL: {total}/{max_total}  ({100*total/max_total:.0f}%)")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    results = run_all_scenarios()
+    print_report(results)

devin/nodes/shared/goldilocks_advisor_eval/scenarios.json

@@ -0,0 +1,58 @@
+[
+  {
+    "id": "small_idea_enrichment_needed",
+    "description": "Idea is too thin — less than 2 stories, should return enrichment_needed",
+    "idea_text": "Add a cancel button to the settings page",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "idea_id": "proj_75f63d30_idea_eval_001",
+    "expected_verdict": "enrichment_needed",
+    "scoring": {
+      "verdict_correct": 2,
+      "story_estimate_present": 1,
+      "enrichment_gaps_filled": 1
+    }
+  },
+  {
+    "id": "medium_idea_just_right",
+    "description": "Well-scoped medium idea in the companion chat domain — should return just_right with estimate 2-20",
+    "idea_text": "Add greeting customization — let users configure different greeting messages per time of day (morning, afternoon, evening) that the companion uses when starting a new conversation session.",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "idea_id": "proj_75f63d30_idea_eval_002",
+    "expected_verdict": "just_right",
+    "scoring": {
+      "verdict_correct": 2,
+      "story_estimate_in_range": 1,
+      "cohesive_area_present": 1
+    }
+  },
+  {
+    "id": "large_idea_split",
+    "description": "Very large multi-domain idea — should return split with split_suggestions",
+    "idea_text": "Rebuild the entire client portal with a new React frontend, PostgreSQL backend, move from REST to GraphQL, implement real-time WebSocket notifications, add multi-tenant billing with Stripe integration, SSO via SAML, and a mobile app via React Native",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "idea_id": "proj_75f63d30_idea_eval_003",
+    "expected_verdict": "split",
+    "scoring": {
+      "verdict_correct": 2,
+      "split_suggestions_present": 2,
+      "story_estimate_over_20": 1
+    }
+  },
+  {
+    "id": "medium_idea_just_right_variation",
+    "description": "Another well-scoped idea in the companion chat domain",
+    "idea_text": "Add persona tone controls so users can switch between formal, casual, and brief response styles for the companion chat.",
+    "repo_root": "/Users/devflow/repos/Spicy-Server",
+    "project_id": "proj_75f63d30",
+    "idea_id": "proj_75f63d30_idea_eval_004",
+    "expected_verdict": "just_right",
+    "scoring": {
+      "verdict_correct": 2,
+      "story_estimate_present": 1,
+      "reasoning_present": 1
+    }
+  }
+]

devin/nodes/shared/helpers.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from devflow_engine.agentic_prompts import load_agentic_prompt_lines, load_agentic_prompt_text
+from devflow_engine.devflow_state import _postgrest_request, _resolve_supabase_rest_config, publish_devflow_state
+from devflow_engine.project_registry import find_project_for_repo_root
+from devflow_engine.stores.execution_store import ExecutionStore
+from devflow_engine.idea.sufficiency import load_idea_source
+
+_REPO_ROOT = Path(__file__).resolve().parents[4]
+DAG_ID = 'devin_chat_dag'
+_CURRENT_STORE: ExecutionStore | None = None
+_CURRENT_RUN_ID: str | None = None
+_SOFTWARE_SIGNAL_TERMS = {'app','application','api','automation','backend','bug','dashboard','feature','frontend','integration','mobile','portal','saas','software','system','tool','ui','web','website','workflow'}
+_INSIGHT_SIGNAL_TERMS = {'status','queue','worker','repo','code','logs','error','why','how','where','what','investigate','explain','architecture','runtime','process','pipeline','see','verify','find','search','look','check','locate'}
+_NON_SOFTWARE_SIGNAL_TERMS = {'blog post','flyer','hiring','logo','marketing','newsletter','podcast','seo','social media','video script'}
+
+@dataclass(frozen=True)
+class DevinChatDagResult:
+    exit_code: int
+    run_id: str
+    pipeline_dir: Path
+    message: str
+    outcome: dict[str, Any]
+
+def set_runtime_store(store: ExecutionStore | None, run_id: str | None) -> None:
+    global _CURRENT_STORE, _CURRENT_RUN_ID
+    _CURRENT_STORE = store
+    _CURRENT_RUN_ID = run_id
+
+def store_run() -> tuple[ExecutionStore, str]:
+    if _CURRENT_STORE is None or _CURRENT_RUN_ID is None:
+        raise RuntimeError('devin chat dag missing runtime store/run_id')
+    return _CURRENT_STORE, _CURRENT_RUN_ID
+
+def write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+def parse_list_option(planes: list[str]) -> list[str]:
+    return sorted({str(item).strip() for item in planes if str(item).strip()})
+
+def stable_id(prefix: str, payload: Any, *, size: int = 12) -> str:
+    digest = hashlib.sha256(json.dumps(payload, sort_keys=True).encode('utf-8')).hexdigest()
+    return f'{prefix}{digest[:size]}'
+
+def build_pipeline_key(*, repo_root: Path, idea_id: str, text: str | None, source_path: Path | None, max_stories: int, planes: list[str]) -> str:
+    raw_text, _ = load_idea_source(text=text, source_path=source_path)
+    return stable_id('run_', {'idea_id': idea_id, 'max_stories': max_stories, 'planes': parse_list_option(planes), 'repo_root': str(repo_root), 'raw_text': raw_text})
+
+def pipeline_root(repo_root: Path, *, idea_id: str, pipeline_key: str) -> Path:
+    return repo_root / '.devflow' / 'ideas' / idea_id / 'pipelines' / DAG_ID / pipeline_key
+
+def resolve_project_id(repo_root: Path, *, idea_id: str) -> str:
+    idea_json = repo_root / '.devflow' / 'ideas' / idea_id / 'idea.json'
+    if idea_json.exists():
+        try:
+            payload = json.loads(idea_json.read_text(encoding='utf-8'))
+            project_id = str(payload.get('project_id') or '').strip()
+            if project_id:
+                return project_id
+        except Exception:
+            pass
+    project_entry = find_project_for_repo_root(repo_root)
+    project_id = None if project_entry is None else str(project_entry.get('project_id') or '').strip()
+    return project_id or f'unregistered:{idea_id}'
+
+def dfs_running(*, project_id: str, run_id: str, summary: str, idea_id: str) -> None:
+    publish_devflow_state(project_id=project_id, run_id=run_id, current_state='running', current_status='processing', run_summary=summary, display='project', display_path=f'idea:{idea_id}')
+
+def dfs_node_running(*, project_id: str, run_id: str, node_id: str, summary: str, idea_id: str) -> None:
+    dfs_running(project_id=project_id, run_id=run_id, summary=f'{summary} [{node_id}]', idea_id=idea_id)
+
+def dfs_terminal(*, project_id: str, run_id: str, summary: str, current_state: str, current_status: str, idea_id: str, error_message: str | None = None) -> None:
+    publish_devflow_state(project_id=project_id, run_id=run_id, current_state=current_state, current_status=current_status, run_summary=summary, error_message=error_message, display='project', display_path=f'idea:{idea_id}')
+
+def _resolve_node_prompt_path(node_file: str) -> Path:
+    node_path = Path(node_file).resolve()
+    local_prompt = node_path.with_name('prompt.md')
+    if local_prompt.exists():
+        return local_prompt
+    parts = list(node_path.parts)
+    try:
+        nodes_idx = parts.index('nodes')
+        node_name = parts[nodes_idx + 1]
+    except (ValueError, IndexError) as exc:
+        raise FileNotFoundError(f'Could not resolve prompt path for node file: {node_file}') from exc
+    shared_prompt = _REPO_ROOT / 'prompts' / 'devin' / node_name / 'prompt.md'
+    if shared_prompt.exists():
+        return shared_prompt
+    raise FileNotFoundError(f'No prompt.md found for node file: {node_file}')
+
+
+def load_node_prompt_text(node_file: str) -> str:
+    return load_agentic_prompt_text(str(_resolve_node_prompt_path(node_file)))
+
+
+def load_node_prompt_lines(node_file: str) -> list[str]:
+    return load_agentic_prompt_lines(str(_resolve_node_prompt_path(node_file)))
+
+def classify_route(raw_text: str) -> tuple[str, dict[str, Any]]:
+    lowered = raw_text.lower()
+    software_hits = sorted(term for term in _SOFTWARE_SIGNAL_TERMS if term in lowered)
+    insight_hits = sorted(term for term in _INSIGHT_SIGNAL_TERMS if term in lowered)
+    non_software_hits = sorted(term for term in _NON_SOFTWARE_SIGNAL_TERMS if term in lowered)
+    if any(token in lowered for token in ('queue', 'worker', 'status', 'repo', 'codebase', 'how does', 'where is', 'why is', 'what is happening')):
+        route_arm = 'insight'; reason = 'project-specific operational or code/explanation request'
+    elif any(phrase in lowered for phrase in ('can you see', 'can you find', 'can you locate', 'please verify', 'can you check', 'do you see', 'do you have the file', 'are the files in')):
+        route_arm = 'insight'; reason = 'repo content verification request'
+    elif software_hits and len(software_hits) >= len(non_software_hits):
+        route_arm = 'ideation'; reason = 'software/product change request'
+    elif any(token in lowered for token in ('build', 'create', 'add', 'implement', 'ship', 'plan')) and not non_software_hits and not insight_hits:
+        route_arm = 'ideation'; reason = 'forward-looking implementation/planning request'
+    elif insight_hits and 'project' in lowered:
+        route_arm = 'insight'; reason = 'project-specific investigation request'
+    else:
+        route_arm = 'neither'; reason = 'not clearly project-specific software delivery work'
+    return route_arm, {'route_arm': route_arm, 'software_signals': software_hits, 'insight_signals': insight_hits, 'non_software_signals': non_software_hits, 'reason': reason, 'classifier': 'deterministic_rules_v1'}
+
+def build_human_facing_message(*, response_payload: dict[str, Any]) -> str:
+    response_message = str(response_payload.get('response_message') or '').strip()
+    next_step = str(response_payload.get('suggested_next_step') or '').strip()
+    if next_step:
+        next_step_sentence = f"Next, you can {next_step[0].lower() + next_step[1:] if next_step and next_step[0].isupper() else next_step}."
+    else:
+        next_step_sentence = ''
+    parts = [part for part in [response_message, next_step_sentence] if part]
+    return (' '.join(parts).strip() or 'I finished that step.') + "\n"
+
+def maybe_post_agent_message(*, row: dict[str, Any]) -> dict[str, Any]:
+    if os.environ.get('PYTEST_CURRENT_TEST'):
+        return {'id': 'test-agent-agent-message', **row}
+    config = _resolve_supabase_rest_config()
+    if config is None:
+        return {'id': 'local-only-agent-agent-message', **row, 'status': 'local_only'}
+    url, key = config
+    table_url = f'{url}/rest/v1/agent_agent_messages'
+    try:
+        result = _postgrest_request(
+            method='POST',
+            url=table_url,
+            key=key,
+            body=row,
+            prefer='return=representation',
+        )
+        if isinstance(result, list) and result:
+            return result[0]
+        return {'id': result.get('id', 'unknown'), **row, 'status': 'posted'}
+    except Exception:
+        return {'id': 'local-only-agent-agent-message', **row, 'status': 'local_only'}