devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/scope_idea/models.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+ScopeShape = Literal["too_broad", "just_right", "too_narrow"]
+ResolutionStatus = Literal["ideas_registered", "narrow_scope_review_required"]
+
+
+class SourceEvidenceRef(BaseModel):
+    ref: str
+    summary: str | None = None
+
+
+class ArtifactLineage(BaseModel):
+    stage: str
+    origin: Literal["model", "fallback", "deterministic"]
+    mode: str
+    artifact_path: str | None = None
+    agent_run_ref: str | None = None
+    generated_from: list[str] = Field(default_factory=list)
+    notes: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class ScopeContextArtifact(BaseModel):
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    scope_title: str
+    scope_description: str
+    source_support: list[SourceEvidenceRef] = Field(default_factory=list)
+    assumptions: list[str] = Field(default_factory=list)
+    cross_cutting_constraints: list[str] = Field(default_factory=list)
+    neighbor_scope_refs: list[str] = Field(default_factory=list)
+    approval_status: str = "approved"
+
+
+class GoldilocksAssessmentArtifact(BaseModel):
+    execution_lineage: ArtifactLineage | None = None
+    scope_shape: ScopeShape
+    reasoning: str
+    dominant_outcomes: list[str] = Field(default_factory=list)
+    workflow_clusters: list[str] = Field(default_factory=list)
+    actor_clusters: list[str] = Field(default_factory=list)
+    outcome_coherence_summary: str | None = None
+    business_outcome_signals: list[str] = Field(default_factory=list)
+    implementation_detail_signals: list[str] = Field(default_factory=list)
+    breadth_signals: list[str] = Field(default_factory=list)
+    cohesion_notes: list[str] = Field(default_factory=list)
+    split_recommended: bool = False
+    recommended_child_idea_count: int = 1
+    merge_review_recommended: bool = False
+    notes: list[str] = Field(default_factory=list)
+
+
+class GoldilocksDecisionArtifact(BaseModel):
+    decision: ScopeShape
+    reason: str
+    next_node: str
+
+
+class IdeaCandidateArtifact(BaseModel):
+    execution_lineage: ArtifactLineage | None = None
+    idea_candidate_id: str
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    parent_scope_id: str
+    refs: list[str] = Field(default_factory=list)
+    title: str
+    problem: str
+    users: list[str]
+    goal: str
+    scope: list[str]
+    constraints: list[str]
+    acceptance_criteria: list[str]
+    assumptions: list[str]
+    traceability: list[str]
+    split_rationale: str | None = None
+
+
+class IdeaSplitPlanArtifact(BaseModel):
+    execution_lineage: ArtifactLineage | None = None
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    parent_scope_id: str
+    refs: list[str] = Field(default_factory=list)
+    split_required: bool
+    split_rationale: str
+    child_ideas: list[IdeaCandidateArtifact]
+    coverage_of_parent_scope: list[str] = Field(default_factory=list)
+    remaining_risks: list[str] = Field(default_factory=list)
+
+
+class IdeaCandidateSetArtifact(BaseModel):
+    lineage: list[ArtifactLineage] = Field(default_factory=list)
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    parent_scope_id: str
+    refs: list[str] = Field(default_factory=list)
+    idea_candidates: list[IdeaCandidateArtifact]
+    candidate_count: int
+    split_applied: bool
+    split_rationale: str | None = None
+
+
+class RegisteredIdeaArtifact(BaseModel):
+    execution_lineage: ArtifactLineage | None = None
+    idea_id: str
+    source_scope_id: str
+    parent_scope_id: str
+    idea_candidate_id: str
+    title: str
+    summary: str
+    registry_ref: str
+    idea_ref: str
+    status: str
+
+
+class IdeaRegistryRecordArtifact(BaseModel):
+    lineage: list[ArtifactLineage] = Field(default_factory=list)
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    parent_scope_id: str
+    refs: list[str] = Field(default_factory=list)
+    scope_shape: ScopeShape
+    registered_ideas: list[RegisteredIdeaArtifact]
+    registration_timestamp: str
+    registry_root: str
+
+
+class NarrowScopeReviewArtifact(BaseModel):
+    execution_lineage: ArtifactLineage | None = None
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    parent_scope_id: str
+    refs: list[str] = Field(default_factory=list)
+    reason_too_narrow: str
+    suggested_merge_targets: list[str] = Field(default_factory=list)
+    human_review_required: bool = True
+    recommended_next_action: str
+
+
+class IdeaResolutionPackageArtifact(BaseModel):
+    lineage: list[ArtifactLineage] = Field(default_factory=list)
+    project_id: str
+    scope_set_id: str
+    scope_id: str
+    parent_scope_id: str
+    refs: list[str] = Field(default_factory=list)
+    scope_shape: ScopeShape
+    split_applied: bool
+    resolution_status: ResolutionStatus
+    registered_idea_count: int
+    registered_ideas: list[RegisteredIdeaArtifact] = Field(default_factory=list)
+    assumptions_added: list[str] = Field(default_factory=list)
+    remaining_risks: list[str] = Field(default_factory=list)
+    recommended_next_action: str
+    review_required: bool = False
+    review_package_ref: str | None = None
+
+
+class ScopeIdeaDagSummary(BaseModel):
+    exit_code: int
+    run_id: str
+    pipeline_dir: str
+    message: str
+    outcome: dict[str, object]

devflow_engine/skills/builtins/devflow/queue_failure_investigation/SKILL.md

@@ -0,0 +1,112 @@
+---
+name: devflow-queue-failure-investigation
+description: Cross-cutting DevFlow skill for failure/log investigation across queue lanes, with log-first evidence order and run-correlation doctrine.
+owner: devflow
+scope: built-in
+version: 1
+---
+
+# DevFlow queue: failure investigation / logs
+
+Use this skill when the task is to answer:
+- what failed
+- where it failed
+- whether the queue state matches the real failure
+- whether recovery is acting on the correct seam
+- whether a run is actually stuck, stale, or merely mis-correlated
+
+This is the cross-cutting investigation module for all queue lanes.
+
+## Core doctrine
+
+Do not start from summary rows alone when richer evidence exists.
+Prefer this evidence order:
+
+1. **Primary:** streamed LLM / agent logs and session evidence
+   - `/Users/devflow/.devflow/llm_logs/*.jsonl`
+   - `/Users/devflow/.devflow/llm_sessions.sqlite`
+2. **Secondary:** node-level artifacts and errors
+   - `artifacts`
+   - `errors`
+   - `nodes`
+3. **Tertiary:** queue rows and worker summary state
+   - `idea_queue`
+   - `story_queue`
+   - `integration_queue`
+   - `recovery_queue`
+   - `project_workers`
+
+## Standard investigation loop
+
+1. Identify the lane and durable item id.
+2. Read the queue row and current worker state.
+3. Correlate to the wrapper/orchestration run.
+4. Search for the real child DAG run if the wrapper is thin.
+5. Read logs/artifacts for the failing stage.
+6. Classify the failure as one of:
+   - real code/runtime failure
+   - stale worker / reconciliation failure
+   - wrong-run investigation
+   - repeated churn with no material change
+   - blocked by prerequisite/state boundary
+
+## Wrapper-vs-child run rule
+
+Never assume the user-facing wrapper run id is the same as the actual child DAG run id.
+Check for mismatch explicitly.
+
+Correlate on:
+- queue item id
+- `story_id` or `idea_id` when relevant
+- repo root
+- created-at time window
+- artifact paths
+- node sequence
+
+A wrapper run can look stranded even while the child DAG succeeded correctly.
+Do not call the whole flow failed until both layers are checked.
+
+## First inspection targets
+
+### For any failed lane
+- queue row `failure_message`
+- queue row `failure_context_json`
+- matching run(s) in `runs`
+- matching node rows in `nodes`
+- matching `errors`
+- high-value `artifacts`
+
+### For story churn
+- `actual_failed_node`
+- `churn_state`
+- streamed transcripts for Green/Refactor/Security/etc.
+- whether the edits hit the correct seam
+
+### For recovery churn
+- recovery diagnosis/execution artifacts
+- durable churn-gate evidence
+- whether the same failure signature repeated without material repo/state change
+
+## What counts as success
+
+A failure investigation is successful when it produces a grounded answer to:
+- the failing boundary
+- the best primary evidence
+- whether the state is real failure vs stale/mis-correlated state
+- the next correct lane: retry, recovery, escalate, or declare blocked
+
+## What counts as blockage
+
+Treat the investigation itself as blocked when:
+- the queue item cannot be correlated to any relevant run or artifact
+- logs are missing and DB evidence is internally inconsistent
+- the worker/report state is stale and must be reconciled first
+- the operator is still looking at the wrong wrapper run
+
+## Strong future tool candidates
+
+- log-first failure packet reader
+- queue item -> run/artifact/session correlation helper
+- wrapper-child mismatch detector
+- churn explainer for story/recovery lanes
+- lane-specific first-evidence summary helper

devflow_engine/skills/builtins/devflow/queue_idea_to_story/SKILL.md

@@ -0,0 +1,120 @@
+---
+name: devflow-queue-idea-to-story
+description: Operational DevFlow skill for starting, observing, and investigating the idea-to-story queue lane.
+owner: devflow
+scope: built-in
+version: 1
+---
+
+# DevFlow queue: idea -> story
+
+Use this skill when the task is about the **runtime lane that turns an already-approved idea into story outputs**.
+
+This skill starts at the queue/runtime boundary.
+It does **not** replace:
+- `devin/ideation`
+- `devin/idea_to_story_handoff`
+
+Those own conversational approval truth.
+This skill owns operational inspection after the idea payload exists or the queue lane is being discussed.
+
+## Core job
+
+Determine whether an idea has:
+- been queued for story generation
+- been claimed and started
+- produced durable story outputs
+- failed, blocked, or stalled
+
+## How to start / kick the lane
+
+Primary runtime path:
+- ensure an `idea_queue` item exists for the `idea_id`
+- run `devflow worker start --project <project_id> --once` for one-step pickup
+- or `devflow worker start --project <project_id>` to drain the queue
+
+Supabase/control-plane path in this repo:
+- `devflow worker supabase-events`
+- supported request event: `devflow.idea.stories.generate.request`
+
+Do not say story generation started just because the idea was approved in conversation.
+Runtime start begins when the queue item is claimed and the worker starts `worker.idea_to_story`.
+
+## Observe status
+
+Check in this order:
+
+1. `devflow worker report --project <project_id>`
+2. `idea_queue` row in `<repo_root>/.devflow/execution.sqlite`
+3. correlated run in `runs` for kind `worker.idea_to_story`
+4. generated idea/story artifacts under `.devflow/ideas/<idea_id>/...`
+5. canonical story outputs under `ai_docs/context/v2/project_docs/user_stories/`
+
+Good first SQL snapshot:
+
+```sql
+select idea_queue_id, idea_id, status, started_run_id, finished_run_id, failure_message, updated_at
+from idea_queue
+where project_id = '<project_id>'
+order by updated_at desc;
+```
+
+High-value artifact locations:
+- `.devflow/ideas/<idea_id>/traditional_user_stories/<story_set_id>/`
+- `.devflow/ideas/<idea_id>/devflow_story_sets/<devflow_story_set_id>/`
+- `ai_docs/context/v2/project_docs/user_stories/`
+
+## Investigate failures
+
+Look first at:
+1. `idea_queue.failure_message`
+2. `idea_queue.failure_context_json`
+3. `runs` / `nodes` / `artifacts` for the correlated run
+4. any `resume_cursor` recorded in failure context
+5. story-generation artifacts in the idea directory
+
+Helpful current commands:
+- `devflow worker failed-ideas --project <project_id>`
+- `devflow worker retry-idea --project <project_id> --latest`
+- `devflow worker retry-idea --project <project_id> --latest --run-now`
+
+If the lane failed after partial output, verify whether the failure happened in:
+- traditional story decomposition
+- story-plane adjudication
+- DevFlow story compilation
+- downstream sync/publication after story artifacts already exist
+
+## Primary evidence to inspect first
+
+1. `idea_queue` row
+2. run + node records
+3. `.devflow/ideas/<idea_id>/traditional_user_stories/...`
+4. `.devflow/ideas/<idea_id>/devflow_story_sets/...`
+5. canonical generated story docs
+
+If richer agent logs exist for the failure, prefer those over summary fields.
+
+## What counts as success
+
+Only treat the lane as successful when:
+- the `idea_queue` row is terminal-complete
+- the run finished successfully
+- durable story outputs exist for the idea
+- the canonical generated story surface was updated when that step belongs to the run
+
+## What counts as blockage
+
+Treat these as blocked/problem states:
+- `idea_queue.status = failed`
+- repeated retries with the same `resume_cursor` and no new durable outputs
+- story artifacts exist but queue/run reconciliation is inconsistent
+- the idea payload is missing or stale for the referenced `idea_id`
+- recovery has taken over the lane
+
+## Strong future tool candidates
+
+- latest `idea_queue` summary for an `idea_id`
+- `resume_cursor` explainer
+- story-output locator for an `idea_id`
+- latest failed idea retry helper
+- queue-row -> run/artifact correlation helper

devflow_engine/skills/builtins/devflow/queue_integration/SKILL.md

@@ -0,0 +1,105 @@
+---
+name: devflow-queue-integration
+description: Operational DevFlow skill for kicking, observing, and investigating integration_queue work and integration DAG outputs.
+owner: devflow
+scope: built-in
+version: 1
+---
+
+# DevFlow queue: integration
+
+Use this skill when the task is about **integration follow-through for an implemented idea**.
+
+## Core job
+
+Determine whether integration work for an `idea_id` has:
+- had a payload prepared
+- been enqueued
+- started via worker execution
+- completed with durable current artifacts
+- failed and handed off to recovery
+
+## How to start / kick the lane
+
+Current repo surfaces:
+- `devflow integration prepare <idea_id>`
+- `devflow integration enqueue <idea_id> --project <project_id>`
+- `devflow worker start --project <project_id> --once`
+
+Manual direct run exists but is not the queue path:
+- `devflow integration run <idea_id>`
+
+When operating the queue lane, prefer enqueue + worker over claiming queue progress from a manual run.
+
+## Observe status
+
+Check in this order:
+
+1. `devflow integration status <idea_id>`
+2. `devflow worker report --project <project_id>`
+3. `integration_queue` row in `.devflow/execution.sqlite`
+4. correlated run of kind `worker.integration`
+5. idea-local integration artifacts
+
+High-value artifact locations:
+- `.devflow/ideas/<idea_id>/integration_payload.json`
+- `.devflow/ideas/<idea_id>/integration/current/`
+- `.devflow/ideas/<idea_id>/integration/runs/`
+- legacy fallback location: `.devflow/ideas/<idea_id>/pipelines/integration_dag/`
+
+Good first SQL snapshot:
+
+```sql
+select integration_queue_id, idea_id, status, started_run_id, finished_run_id, failure_message, updated_at
+from integration_queue
+where project_id = '<project_id>'
+order by updated_at desc;
+```
+
+## Investigate failures
+
+Look first at:
+1. `integration_queue.failure_message`
+2. `integration_queue.failure_context_json`
+3. `integration_payload.json`
+4. latest pipeline run directory
+5. run/node/artifact/error rows for `worker.integration`
+6. recovery queue state if the failure already escalated
+
+Common real failure shapes:
+- payload missing or stale for the `idea_id`
+- implemented-story evidence incomplete
+- integration DAG failed after starting
+- queue row failed but current artifacts partially exist
+
+## Primary evidence to inspect first
+
+1. `integration_queue` row
+2. `devflow integration status <idea_id>` output
+3. payload path
+4. latest integration run dir
+5. correlated DB run/artifact/error records
+
+## What counts as success
+
+Only treat the lane as successful when:
+- `integration_queue.status = completed`
+- the integration run succeeded
+- `.devflow/ideas/<idea_id>/integration/current/` exists or equivalent latest durable current state exists
+- any registered-project sync side effects completed when expected
+
+## What counts as blockage
+
+Treat these as blocked/problem states:
+- queue row failed
+- payload cannot be prepared/resolved
+- latest run exists but current durable integration state was not updated
+- recovery is now the active owner of the failed integration item
+
+## Strong future tool candidates
+
+- `idea_id` -> integration status summary
+- integration payload validator/locator
+- latest integration run summary
+- integration queue retry / inspect helper
+- queue-to-artifact correlation helper

devflow_engine/skills/builtins/devflow/queue_recovery/SKILL.md

@@ -0,0 +1,108 @@
+---
+name: devflow-queue-recovery
+description: Operational DevFlow skill for recovery_queue execution, recovery outcome classification, and recovery handoff inspection.
+owner: devflow
+scope: built-in
+version: 1
+---
+
+# DevFlow queue: recovery
+
+Use this skill when the task is about **post-failure recovery work** in `recovery_queue`.
+
+This skill complements `docs/recovery-dag-contract.md`.
+That doc explains the DAG contract.
+This skill explains how to operate and inspect the recovery lane concretely.
+
+## Core job
+
+Determine whether a failed queue item has:
+- been handed off into `recovery_queue`
+- started a real recovery run
+- been re-enqueued, delegated, blocked, or fully recovered
+- produced the expected recovery handoff artifact
+
+## How to start / kick the lane
+
+Normal path:
+- recovery items are enqueued automatically when a queue lane fails
+- run `devflow worker start --project <project_id> --once` or drain the worker
+
+Separate command with different purpose:
+- `devflow worker recover --project <project_id>` repairs **stale worker ownership state**
+- it does **not** itself run the business recovery DAG for a fresh failure
+
+Do not bypass recovery state with direct SQL edits unless the task is explicitly DB repair.
+
+## Observe status
+
+Check in this order:
+
+1. `devflow worker report --project <project_id>`
+2. `recovery_queue` row in `.devflow/execution.sqlite`
+3. correlated recovery run, usually `post_queue_failure_recovery_dag`
+4. recovery execution metadata / artifacts
+5. recovery handoff artifact on disk
+6. source queue row after recovery action
+
+Good first SQL snapshot:
+
+```sql
+select recovery_queue_id, source_queue_type, source_item_id, status, started_run_id, finished_run_id, failure_message, updated_at
+from recovery_queue
+where project_id = '<project_id>'
+order by updated_at desc;
+```
+
+High-value handoff artifact locations:
+- story-scoped: `.devflow/stories/<story_id>/recovery_handoff.json`
+- non-story fallback: `.devflow/recovery_handoffs/<queue_type>/<item_id>.json`
+
+## Investigate failures
+
+Look first at:
+1. `recovery_queue` row
+2. correlated recovery run + node/artifact/error records
+3. recovery handoff artifact
+4. source queue item state after attempted recovery
+5. rich logs gathered during recovery investigation/execution
+
+Important recovery outcomes to classify correctly:
+- `reenqueued`
+- `delegated`
+- `blocked`
+- `recovered`
+
+Do not compress those into a generic "fixed" claim.
+
+## Primary evidence to inspect first
+
+1. `recovery_queue.status`
+2. recovery run status
+3. published recovery summary / handoff artifact
+4. source queue row reconciliation
+5. churn/no-material-change evidence if retries repeat
+
+## What counts as success
+
+Only treat recovery as successful when the durable result matches the declared outcome:
+- `reenqueued` -> source queue row was actually reset/re-queued
+- `delegated` -> the handoff/delegation artifact exists and the target lane is explicit
+- `recovered` -> the recovery row completed and the source item is reconciled as intended
+
+## What counts as blockage
+
+Treat these as blocked/problem states:
+- `recovery_queue.status = failed` or `blocked`
+- durable churn gate fired for repeated no-material-change recovery
+- recovery kept acting on the wrong seam
+- handoff artifact is missing despite a claimed publish step
+- source queue row did not reconcile to the declared outcome
+
+## Strong future tool candidates
+
+- latest recovery queue item summary
+- source queue item -> recovery chain reader
+- recovery handoff locator
+- recovery outcome explainer
+- durable churn-gate state reader