devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["__version__"]
+
+__version__ = "1.0.0"

devflow_engine/agentic_prompts.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from importlib.resources import files
+from pathlib import Path
+from typing import Any
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_PROMPTS_ROOT = _REPO_ROOT / "prompts"
+
+_PROMPT_LAYOUT: dict[str, tuple[str, str]] = {
+    "idea_response_doctrine": ("idea", "response_doctrine"),
+    "idea_api_ideation_agent": ("idea", "api_ideation_agent"),
+    "idea_api_insight_agent": ("idea", "api_insight_agent"),
+    "idea_generic_response_agent": ("devin", "generic"),
+    "idea_insight_response_agent": ("devin", "insight"),
+    "idea_ideation_response_agent": ("devin", "ideation"),
+    "idea_devin_ideation_agent_loop": ("devin", "ideation_loop"),
+    "source_scope_doctrine": ("source_scope", "doctrine"),
+    "scope_idea_doctrine": ("scope_idea", "doctrine"),
+    "ui_grounding_doctrine": ("ui_grounding", "doctrine"),
+    "implementation_dependency_assessment": ("implementation", "dependency_assessment"),
+    "implementation_story_planning": ("implementation", "story_planning"),
+    "implementation_test_design": ("implementation", "test_design"),
+    "implementation_setupdoc": ("implementation", "setupdoc"),
+    "implementation_red": ("implementation", "red"),
+    "implementation_redreview": ("implementation", "redreview"),
+    "implementation_redreview_repair": ("implementation", "redreview_repair"),
+    "recovery_diagnosis": ("recovery", "diagnosis"),
+    "recovery_preflight_health_repo_repair": ("recovery", "preflight_health_repo_repair"),
+    "recovery_failure_investigation": ("recovery", "failure_investigation"),
+    "recovery_root_cause_investigation": ("recovery", "root_cause_investigation"),
+    "recovery_remediation_execution": ("recovery", "remediation_execution"),
+    "recovery_execution": ("recovery", "execution"),
+    "recovery_execution_verification": ("recovery", "execution_verification"),
+    "source_doc_mutation_project_doc_render": ("source_doc_mutation", "project_doc_render"),
+    "source_doc_mutation_product_brief": ("source_doc_mutation", "product_brief"),
+    "source_doc_mutation_user_workflows": ("source_doc_mutation", "user_workflows"),
+    "source_doc_mutation_domain_entities": ("source_doc_mutation", "domain_entities"),
+    "source_doc_mutation_source_doc_coherence": ("source_doc_mutation", "source_doc_coherence"),
+    "source_doc_mutation_source_doc_enrichment_coherence": ("source_doc_mutation", "source_doc_enrichment_coherence"),
+    "source_doc_mutation_project_doc_coherence": ("source_doc_mutation", "project_doc_coherence"),
+    "devin_eval_assessment": ("devin_eval", "assessment"),
+    "source_doc_eval_document": ("source_doc_eval", "document"),
+    "source_doc_eval_targeted_mutation": ("source_doc_eval", "targeted_mutation"),
+}
+
+
+def agentic_prompt_path(prompt_name: str) -> Path:
+    direct = Path(prompt_name).expanduser()
+    if direct.suffix.lower() == ".md":
+        if not direct.is_absolute():
+            direct = (_REPO_ROOT / direct).resolve()
+        return direct
+    try:
+        group, agent_name = _PROMPT_LAYOUT[prompt_name]
+    except KeyError as exc:
+        raise KeyError(f"Unknown agentic prompt mapping: {prompt_name}") from exc
+    source_path = _PROMPTS_ROOT / group / agent_name / "prompt.md"
+    if source_path.exists():
+        return source_path
+    return Path(str(_packaged_prompts_root().joinpath(group, agent_name, "prompt.md")))
+
+
+def _packaged_prompts_root() -> Any:
+    return files("devflow_engine").joinpath("prompts")
+
+
+def _prompt_text(path: Path) -> str:
+    if path.exists():
+        return path.read_text(encoding="utf-8").strip()
+    package_path = _packaged_prompts_root().joinpath(*path.parts[-3:])
+    return package_path.read_text(encoding="utf-8").strip()
+
+
+def load_agentic_prompt_markdown(prompt_name: str) -> str:
+    return _prompt_text(agentic_prompt_path(prompt_name))
+
+
+def load_agentic_prompt_lines(prompt_name: str) -> list[str]:
+    items = _parse_prompt_markdown(load_agentic_prompt_markdown(prompt_name))
+    if not items:
+        raise ValueError(f"Agentic prompt markdown is empty: {prompt_name}")
+    return items
+
+
+def load_agentic_prompt_text(prompt_name: str) -> str:
+    return "\n".join(load_agentic_prompt_lines(prompt_name))
+
+
+def _parse_prompt_markdown(markdown: str) -> list[str]:
+    items: list[str] = []
+    for raw_line in markdown.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("- "):
+            items.append(line[2:].strip())
+            continue
+        items.append(line)
+    return items

devflow_engine/agentic_runtime.py

@@ -0,0 +1,398 @@
+from __future__ import annotations
+
+import json
+import re
+from json import JSONDecodeError
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ValidationError
+
+from .idea.traditional_stories import _extract_json, _global_devflow_dir, _read_toml
+from .llm.invoke import (
+    LlmInvocationRequest,
+    _apply_model_flag,
+    _load_cli_config_from_cfg,
+    invoke_llm,
+    normalize_llm_cli_base,
+)
+
+
+def _resolve_strength_model(
+    cfg: dict[str, Any],
+    base_cmd: str,
+    *,
+    strength_override: str,
+) -> str | None:
+    """Resolve the CLI model for a given strength tier from the TOML config.
+
+    Looks for ``llm_cli_model_<strength>`` in cfg (e.g. llm_cli_model_strong).
+    Returns None if the key is absent or empty — caller must decide how to
+    handle the missing value (e.g. apply tier cliProfile separately).
+    Strips incidental whitespace from the resolved value.
+    """
+    key = f"llm_cli_model_{strength_override}"
+    raw = str(cfg.get(key) or "").strip()
+    return raw if raw else None
+
+# ── Prompt size limits ───────────────────────────────────────────────────────
+_MAX_PROMPT_CHARS = 120_000
+_MAX_STRING_CHARS = 8_000
+_MAX_LIST_ITEMS = 40
+_MAX_DICT_ITEMS = 80
+
+
+# ── Prompt shrinking ─────────────────────────────────────────────────────────
+
+def _truncate_string(value: str, *, limit: int = _MAX_STRING_CHARS) -> str:
+    if len(value) <= limit:
+        return value
+    half = limit // 2
+    return value[:half] + f"...[{len(value) - limit} chars truncated]..." + value[-half:]
+
+
+def _shrink_for_prompt(value: Any) -> Any:
+    """Recursively truncate large strings/lists/dicts to keep prompts within context limits."""
+    if isinstance(value, str):
+        return _truncate_string(value)
+    if isinstance(value, dict):
+        items = list(value.items())
+        if len(items) > _MAX_DICT_ITEMS:
+            items = items[:_MAX_DICT_ITEMS]
+        return {k: _shrink_for_prompt(v) for k, v in items}
+    if isinstance(value, list):
+        if len(value) > _MAX_LIST_ITEMS:
+            value = value[:_MAX_LIST_ITEMS]
+        return [_shrink_for_prompt(item) for item in value]
+    return value
+
+
+# ── Prompt construction ──────────────────────────────────────────────────────
+
+def _build_prompt(
+    *,
+    stage_name: str,
+    context_payload: dict[str, Any],
+    guidance: list[str],
+    output_model: type[BaseModel],
+    compact: bool = False,
+) -> dict[str, Any]:
+    ctx = _shrink_for_prompt(context_payload) if compact else context_payload
+    return {
+        "task": stage_name,
+        "instructions": guidance,
+        "context": ctx,
+        "output_schema": output_model.model_json_schema(),
+        "return_format": "json_only",
+    }
+
+
+def _persist_large_context_artifact(
+    *,
+    repo_root: Path,
+    stage_name: str,
+    context_payload: dict[str, Any],
+) -> Path:
+    debug_root = repo_root / ".devflow" / "agent_debug" / "context_artifacts"
+    debug_root.mkdir(parents=True, exist_ok=True)
+    safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", stage_name) or "stage"
+    artifact_path = debug_root / f"{safe}_context.json"
+    artifact_path.write_text(
+        json.dumps(context_payload, indent=2, sort_keys=True), encoding="utf-8"
+    )
+    return artifact_path
+
+
+def _build_artifact_reference_prompt(
+    *,
+    stage_name: str,
+    output_model: type[BaseModel],
+    guidance: list[str],
+    artifact_path: Path,
+) -> dict[str, Any]:
+    return {
+        "task": stage_name,
+        "instructions": guidance,
+        "context_artifact_path": str(artifact_path),
+        "context_artifact_note": (
+            "The full context was persisted to disk because the prompt exceeded size limits. "
+            "Read the file at the path above for detailed context."
+        ),
+        "output_schema": output_model.model_json_schema(),
+        "return_format": "json_only",
+    }
+
+
+# ── JSON extraction and repair ───────────────────────────────────────────────
+
+def _extract_json_hardened(stdout: str) -> str | None:
+    """Hardened JSON extraction — falls back to greedy brace-matching when the
+    standard extractor finds nothing."""
+    result = _extract_json(stdout)
+    if result is not None:
+        return result
+    # Greedy scan: find the last valid JSON object in the output.
+    for match in reversed(list(re.finditer(r"\{", stdout))):
+        candidate = stdout[match.start():]
+        try:
+            json.loads(candidate)
+            return candidate
+        except JSONDecodeError:
+            pass
+    return None
+
+
+def _persist_non_json_response(
+    *,
+    repo_root: Path,
+    stage_name: str,
+    stdout: str,
+    stderr: str,
+) -> Path:
+    debug_root = repo_root / ".devflow" / "agent_debug" / "non_json_responses"
+    debug_root.mkdir(parents=True, exist_ok=True)
+    safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", stage_name) or "stage"
+    artifact_path = debug_root / f"{safe}_response.txt"
+    artifact_path.write_text(
+        f"STDOUT:\n{stdout}\n\nSTDERR:\n{stderr}", encoding="utf-8"
+    )
+    return artifact_path
+
+
+def _repair_json_decode_error(
+    *,
+    repo_root: Path,
+    base_cmd: str,
+    delivery: str,
+    stage_name: str,
+    output_model: type[BaseModel],
+    raw_candidate: str,
+    decode_error: JSONDecodeError,
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, dict[str, Any], Any] | None:
+    """Re-prompt the LLM to repair a malformed JSON response."""
+    repair_prompt: dict[str, Any] = {
+        "task": f"{stage_name}_json_repair",
+        "instructions": [
+            "Return JSON only. No markdown. No prose outside JSON.",
+            "The previous response contained malformed JSON. Repair it and return valid JSON only.",
+            f"JSON parse error: {str(decode_error)[:500]}",
+        ],
+        "malformed_json": raw_candidate[:4000],
+        "output_schema": output_model.model_json_schema(),
+        "return_format": "json_only",
+    }
+    try:
+        result = invoke_llm(LlmInvocationRequest(
+            purpose=f"{stage_name}_json_repair",
+            repo_root=repo_root,
+            prompt=json.dumps(repair_prompt, indent=2, sort_keys=True),
+            delivery_model="streaming",
+            interaction_model="agentic",
+            timeout_seconds=timeout_seconds,
+            base_cmd=base_cmd,
+            delivery=delivery,
+            strength="medium",
+        ))
+        if not result.ok:
+            return None
+        raw_json = _extract_json_hardened(result.stdout)
+        if raw_json is None:
+            return None
+        parsed = json.loads(raw_json)
+        model = output_model.model_validate(parsed)
+        return model, repair_prompt, result
+    except Exception:
+        return None
+
+
+def _repair_validation_error(
+    *,
+    repo_root: Path,
+    base_cmd: str,
+    delivery: str,
+    stage_name: str,
+    output_model: type[BaseModel],
+    original_prompt: dict[str, Any],
+    raw_json: str,
+    validation_error: ValidationError,
+    timeout_seconds: int | None,
+) -> tuple[BaseModel, dict[str, Any], Any] | None:
+    """Re-prompt the LLM to repair a response that failed Pydantic validation."""
+    repair_prompt: dict[str, Any] = {
+        "task": f"{stage_name}_validation_repair",
+        "instructions": [
+            "Return JSON only. No markdown. No prose outside JSON.",
+            "The previous response failed schema validation. Fix the JSON to match the required schema.",
+            f"Validation error: {str(validation_error)[:1000]}",
+        ],
+        "invalid_json": raw_json[:4000],
+        "output_schema": output_model.model_json_schema(),
+        "return_format": "json_only",
+    }
+    try:
+        result = invoke_llm(LlmInvocationRequest(
+            purpose=f"{stage_name}_validation_repair",
+            repo_root=repo_root,
+            prompt=json.dumps(repair_prompt, indent=2, sort_keys=True),
+            delivery_model="streaming",
+            interaction_model="agentic",
+            timeout_seconds=timeout_seconds,
+            base_cmd=base_cmd,
+            delivery=delivery,
+            strength="medium",
+        ))
+        if not result.ok:
+            return None
+        raw_json2 = _extract_json_hardened(result.stdout)
+        if raw_json2 is None:
+            return None
+        parsed = json.loads(raw_json2)
+        model = output_model.model_validate(parsed)
+        return model, repair_prompt, result
+    except Exception:
+        return None
+
+
+class AgentRunEnvelope(BaseModel):
+    prompt: dict[str, Any]
+    response: dict[str, Any]
+    raw_stdout: str
+    raw_stderr: str = ""
+
+
+def _load_llm_cli_config() -> tuple[str, str]:
+    """Compatibility wrapper kept in agentic_runtime for existing callers/tests."""
+    cfg = _read_toml(_global_devflow_dir() / "config.toml")
+    _, base_cmd, delivery = _load_cli_config_from_cfg(cfg)
+    return base_cmd, delivery
+
+
+# ── Core agent step ──────────────────────────────────────────────────────────
+
+def run_agent_step(
+    *,
+    repo_root: Path,
+    stage_name: str,
+    output_model: type[BaseModel],
+    context_payload: dict[str, Any],
+    guidance: list[str],
+    timeout_seconds: int | None = None,
+    strength: str | None = None,
+) -> tuple[BaseModel, AgentRunEnvelope]:
+    """Execute one GenAI agent step with prompt-size management and output repair."""
+    # Build prompt — shrink if needed, fall back to artifact reference for huge contexts.
+    prompt = _build_prompt(
+        stage_name=stage_name,
+        context_payload=context_payload,
+        guidance=guidance,
+        output_model=output_model,
+    )
+    if len(json.dumps(prompt)) > _MAX_PROMPT_CHARS:
+        compact_prompt = _build_prompt(
+            stage_name=stage_name,
+            context_payload=context_payload,
+            guidance=guidance,
+            output_model=output_model,
+            compact=True,
+        )
+        if len(json.dumps(compact_prompt)) <= _MAX_PROMPT_CHARS:
+            prompt = compact_prompt
+        else:
+            artifact_path = _persist_large_context_artifact(
+                repo_root=repo_root,
+                stage_name=stage_name,
+                context_payload=context_payload,
+            )
+            prompt = _build_artifact_reference_prompt(
+                stage_name=stage_name,
+                output_model=output_model,
+                guidance=guidance,
+                artifact_path=artifact_path,
+            )
+
+    result = invoke_llm(
+        LlmInvocationRequest(
+            purpose=stage_name,
+            repo_root=repo_root,
+            prompt=json.dumps(prompt, indent=2, sort_keys=True),
+            prompt_payload=prompt,
+            delivery_model="streaming",
+            interaction_model="agentic",
+            response_contract="json_only",
+            timeout_seconds=timeout_seconds,
+            strength=strength,
+        )
+    )
+
+    if not result.ok:
+        raise RuntimeError(result.stderr or result.stdout or f"agent step failed: {stage_name}")
+
+    raw_json = _extract_json_hardened(result.stdout)
+    if raw_json is None:
+        artifact_path = _persist_non_json_response(
+            repo_root=repo_root,
+            stage_name=stage_name,
+            stdout=result.stdout,
+            stderr=result.stderr,
+        )
+        raise RuntimeError(
+            f"Failed to locate JSON in agent output for {stage_name}. "
+            f"Non-JSON response persisted to {artifact_path}"
+        )
+
+    # Parse with repair fallback.
+    try:
+        parsed = json.loads(raw_json)
+    except JSONDecodeError as exc:
+        repaired = _repair_json_decode_error(
+            repo_root=repo_root,
+            base_cmd=result.base_cmd,
+            delivery=result.delivery,
+            stage_name=stage_name,
+            output_model=output_model,
+            raw_candidate=raw_json,
+            decode_error=exc,
+            timeout_seconds=timeout_seconds,
+        )
+        if repaired is not None:
+            model, repair_prompt, repair_result = repaired
+            return model, AgentRunEnvelope(
+                prompt=repair_prompt,
+                response=model.model_dump(),
+                raw_stdout=repair_result.stdout,
+                raw_stderr=repair_result.stderr,
+            )
+        raise RuntimeError(f"JSON decode error in {stage_name} and repair failed: {exc}") from exc
+
+    # Validate with repair fallback.
+    try:
+        model = output_model.model_validate(parsed)
+    except ValidationError as exc:
+        repaired = _repair_validation_error(
+            repo_root=repo_root,
+            base_cmd=result.base_cmd,
+            delivery=result.delivery,
+            stage_name=stage_name,
+            output_model=output_model,
+            original_prompt=prompt,
+            raw_json=raw_json,
+            validation_error=exc,
+            timeout_seconds=timeout_seconds,
+        )
+        if repaired is not None:
+            model, repair_prompt, repair_result = repaired
+            return model, AgentRunEnvelope(
+                prompt=repair_prompt,
+                response=model.model_dump(),
+                raw_stdout=repair_result.stdout,
+                raw_stderr=repair_result.stderr,
+            )
+        raise RuntimeError(f"Validation error in {stage_name} and repair failed: {exc}") from exc
+
+    return model, AgentRunEnvelope(
+        prompt=prompt,
+        response=model.model_dump(),
+        raw_stdout=result.stdout,
+        raw_stderr=result.stderr,
+    )