devflow-engine 1.1.3__tar.gz → 1.1.4__tar.gz

{devflow_engine-1.1.3 → devflow_engine-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devflow-engine
-Version: 1.1.3
+Version: 1.1.4
 Summary: DevFlow v2 CLI-first engine (projects, stories, execution store, review packets)
 Author: DevFlow
 License: Proprietary

{devflow_engine-1.1.3 → devflow_engine-1.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "devflow-engine"
-version = "1.1.3"
+version = "1.1.4"
 description = "DevFlow v2 CLI-first engine (projects, stories, execution store, review packets)"
 readme = "README.md"
 requires-python = ">=3.11"

{devflow_engine-1.1.3 → devflow_engine-1.1.4}/src/devflow_engine/errors/user_report_observability.py

@@ -280,12 +280,17 @@ def _build_prompt_payload(
         "instruction": (
             "You are the first-pass observability analyzer for a user-reported bug in "
             "a real codebase. Read the report and the supplied repository context, then "
-            "emit a single JSON object matching the supplied output_schema. Be concrete: "
-            "every entry in affected_files needs a workspace-relative path; every "
-            "repro_test needs a plausible filename and language. Use 'open_questions' "
-            "for anything you can't answer from the supplied context — do NOT guess. "
-            "Confidence is your honest signal: 'high' only when you can name the function "
-            "and reproduction steps, 'low' when open_questions has multiple items."
+            "emit a SINGLE JSON OBJECT matching the supplied output_schema. The response "
+            "MUST be one top-level object with these keys: summary (string), "
+            "affected_files (list), likely_location (string|null), repro_tests (list), "
+            "open_questions (list), confidence ('low'|'medium'|'high'). Do NOT return a "
+            "bare list of file hits — that's the affected_files VALUE, not the whole "
+            "response. Be concrete: every affected_files entry needs a workspace-relative "
+            "path + reason; every repro_test needs a plausible filename and language. "
+            "Use 'open_questions' for anything you can't answer from the supplied "
+            "context — do NOT guess. Confidence is your honest signal: 'high' only when "
+            "you can name the function and reproduction steps, 'low' when open_questions "
+            "has multiple items."
         ),
         "error_report": {
             "title": error.get("title"),
@@ -306,6 +311,48 @@ def _build_prompt_payload(
     }
 
 
+def _looks_like_file_hit(item: Any) -> bool:
+    """Heuristic: a dict with both 'path' and 'reason' string fields is a
+    FileHit-shaped record. Used to recognize the model's frequent
+    bare-list-of-affected-files near-miss against ObservabilityOutput."""
+    return (
+        isinstance(item, dict)
+        and isinstance(item.get("path"), str)
+        and isinstance(item.get("reason"), str)
+    )
+
+
+def _normalize_llm_payload(parsed: Any) -> tuple[Any, bool]:
+    """Return ``(payload, was_wrapped)``. If `parsed` is already a dict
+    we pass it through unchanged. If it's a list of FileHit-shaped
+    dicts (the most common near-miss MiniMax emits), we wrap it as
+    `affected_files` with low confidence and an explicit open_question
+    saying so — the tolerance is visible in the output, not hidden."""
+    if isinstance(parsed, dict):
+        return parsed, False
+    if isinstance(parsed, list) and parsed and all(_looks_like_file_hit(it) for it in parsed):
+        return (
+            {
+                "summary": (
+                    "Model returned a bare list of affected files without the "
+                    "ObservabilityOutput envelope; wrapped client-side. "
+                    "Re-run for a full analysis."
+                ),
+                "affected_files": parsed,
+                "likely_location": None,
+                "repro_tests": [],
+                "open_questions": [
+                    "Model response was a bare list and missed the response "
+                    "envelope — re-run, or tighten the prompt's wrapper "
+                    "requirement, to get summary / likely_location / repro_tests.",
+                ],
+                "confidence": "low",
+            },
+            True,
+        )
+    return parsed, False
+
+
 def _call_llm(*, repo_root: Path, prompt_payload: dict[str, Any]) -> ObservabilityOutput | str:
     """Returns ObservabilityOutput on success or a string error reason on failure."""
     try:
@@ -336,8 +383,9 @@ def _call_llm(*, repo_root: Path, prompt_payload: dict[str, Any]) -> Observabili
         return f"llm_not_ok: rc={result.returncode} stderr={(result.stderr or '')[:300]}"
     if not result.contract_ok or result.parsed_json is None:
         return f"llm_contract_failed: {(result.contract_error or result.stdout or '')[:300]}"
+    normalized, _ = _normalize_llm_payload(result.parsed_json)
     try:
-        return ObservabilityOutput.model_validate(result.parsed_json)
+        return ObservabilityOutput.model_validate(normalized)
     except Exception as exc:
         return f"output_schema_validation_failed: {exc}"
 

devflow_engine-1.1.4/tests/area8_errors/test_user_report_observability_llm_output_shapes.py

@@ -0,0 +1,126 @@
+"""Coverage for the LLM-output normalization in user_report_observability.
+
+MiniMax (and other models) don't reliably wrap structured output in the
+declared schema envelope. The analyzer has to be tolerant of common
+near-miss shapes — a bare list of file-hit dicts, a dict missing the
+summary, etc. — without ever silently fabricating data that wasn't in
+the model response.
+
+These tests pin three contracts:
+
+1. A bare list of FileHit-shaped dicts gets wrapped as
+   `affected_files` with a low-confidence summary that says exactly
+   what happened. The tolerance is visible in the output; nothing is
+   invented.
+
+2. A dict missing optional fields still parses; defaults match
+   ObservabilityOutput's declared defaults.
+
+3. A response that's neither a list of file hits NOR a parseable dict
+   surfaces as `output_schema_validation_failed` — the analyzer doesn't
+   manufacture a fake-success.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from devflow_engine.errors import user_report_observability as obs
+
+
+def _run_with_fake_llm_response(parsed_json, monkeypatch, tmp_path: Path):
+    """Helper: run the analyzer against an in-memory store with a fake
+    LLM that returns the given `parsed_json` as the structured response.
+    Returns the ObservabilityRunResult."""
+    from devflow_engine.stores.execution_store import ExecutionStore
+    store = ExecutionStore(db_path=tmp_path / "devflow.db")
+    eid = store.create_error_task_from_user_report(
+        project_id="t", source_app="x", external_id="e1",
+        title="t", message="m",
+    )
+    fake_result = type("FakeResult", (), {
+        "ok": True,
+        "contract_ok": True,
+        "parsed_json": parsed_json,
+        "stdout": "", "stderr": "", "returncode": 0,
+        "contract_error": None,
+    })()
+    monkeypatch.setattr(obs, "invoke_llm", lambda req: fake_result)
+    return store, obs.build_user_report_observability(
+        store=store, repo_root=tmp_path, error_task_id=eid,
+    )
+
+
+def test_bare_list_of_file_hits_is_wrapped_into_envelope(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    """LLM commonly returns just the affected_files list. The analyzer
+    must accept that and produce an ObservabilityOutput-shaped envelope
+    with low confidence and an open question explaining the wrap."""
+    parsed = [
+        {"path": "backend/app/routes/registrations.py",
+         "reason": "POST registration handler — likely returns 200 but doesn't persist."},
+        {"path": "frontend/src/components/MeetingCard.tsx",
+         "reason": "Toast triggered before server confirms persistence."},
+    ]
+    store, res = _run_with_fake_llm_response(parsed, monkeypatch, tmp_path)
+    assert res.status == "observed", res.error_reason
+    output = res.output
+    assert output is not None
+    # The list got wrapped — affected_files preserved verbatim
+    assert len(output["affected_files"]) == 2
+    assert output["affected_files"][0]["path"].endswith("registrations.py")
+    # Confidence MUST be low (we didn't get a summary from the model)
+    assert output["confidence"] == "low"
+    # The wrap must be explicit, not silent
+    assert any("list" in q.lower() or "wrapper" in q.lower() or "envelope" in q.lower()
+               for q in output["open_questions"]), (
+        f"wrap is not surfaced in open_questions; got {output['open_questions']!r}"
+    )
+
+
+def test_dict_missing_optional_fields_still_parses(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    """If the model returns a partial dict — summary + affected_files
+    only — the analyzer fills declared defaults for the missing fields
+    rather than failing or fabricating values."""
+    parsed = {
+        "summary": "Toast fires before persistence completes.",
+        "affected_files": [
+            {"path": "backend/app/routes/registrations.py",
+             "reason": "registration POST handler"},
+        ],
+    }
+    store, res = _run_with_fake_llm_response(parsed, monkeypatch, tmp_path)
+    assert res.status == "observed", res.error_reason
+    output = res.output
+    assert output is not None
+    assert output["summary"].startswith("Toast")
+    assert output["repro_tests"] == []          # declared default
+    assert output["open_questions"] == []       # declared default
+    assert output["confidence"] == "low"        # declared default
+
+
+def test_garbage_response_surfaces_as_validation_failure(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    """If the model returns something we can't sensibly wrap (e.g. a
+    bare string, or a list of dicts that aren't FileHit-shaped), the
+    analyzer must surface output_schema_validation_failed in
+    error_reason — NOT silently create a synthetic empty envelope."""
+    parsed = ["this", "is", "not", "an", "analysis"]
+    store, res = _run_with_fake_llm_response(parsed, monkeypatch, tmp_path)
+    assert res.status == "open", "garbage response must NOT flip to observed"
+    assert res.error_reason is not None
+    assert "validation" in res.error_reason.lower(), res.error_reason
+    # observability_json must NOT be set on the row
+    with store._connect() as conn:
+        row = conn.execute(
+            "SELECT status, observability_json FROM error_tasks LIMIT 1",
+        ).fetchone()
+    assert row["status"] == "open"
+    assert row["observability_json"] is None

{devflow_engine-1.1.3 → devflow_engine-1.1.4}/uv.lock

@@ -172,7 +172,7 @@ wheels = [
 
 [[package]]
 name = "devflow-engine"
-version = "0.1.0"
+version = "1.1.3"
 source = { editable = "." }
 dependencies = [
     { name = "cryptography" },