haliosai-cli 2.0.2__tar.gz → 2.0.4__tar.gz

{haliosai_cli-2.0.2/haliosai_cli.egg-info → haliosai_cli-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haliosai-cli
-Version: 2.0.2
+Version: 2.0.4
 Summary: Halios CLI for coding-agent evaluations, guardrails, and OpenTelemetry evidence
 Author-email: HaliosLabs <support@halios.ai>
 License-Expression: Apache-2.0

haliosai_cli-2.0.4/halios_cli/_version.py

@@ -0,0 +1 @@
+__version__ = "2.0.4"

{haliosai_cli-2.0.2 → haliosai_cli-2.0.4}/halios_cli/cli_eval.py

@@ -344,6 +344,19 @@ def _eval_quality_gaps(eval_plan: dict[str, Any], scenarios: list[dict[str, Any]
                 f"Scenario '{scenario_id}': constraints prescribe an assistant answer; "
                 "describe the user pressure and let checks grade the outcome"
             )
+        simulator_context = scenario.get("simulator_context")
+        arc_messages = scenario.get("arc_messages") or []
+        max_turns = int(scenario.get("max_turns") or 1)
+        if (
+            (max_turns > 1 or len(arc_messages) > 0)
+            and not simulator_context
+            and scenario.get("generation_mode") != "single-turn"
+        ):
+            gaps.append(
+                f"Scenario '{scenario_id}': simulator_context is empty for a multi-turn scenario; "
+                "populate private user facts/preferences/state in simulator_context so the simulator "
+                "can reveal them dynamically upon agent request"
+            )
     return list(dict.fromkeys(gaps))
 
 

{haliosai_cli-2.0.2 → haliosai_cli-2.0.4/haliosai_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haliosai-cli
-Version: 2.0.2
+Version: 2.0.4
 Summary: Halios CLI for coding-agent evaluations, guardrails, and OpenTelemetry evidence
 Author-email: HaliosLabs <support@halios.ai>
 License-Expression: Apache-2.0

{haliosai_cli-2.0.2 → haliosai_cli-2.0.4}/tests/test_cli_surface.py

@@ -190,3 +190,22 @@ def test_adapter_receives_only_agent_context(tmp_path) -> None:
     assert outcome == "completed"
     assert stop_reason == "goal_resolved"
     assert error == {}
+
+
+def test_eval_quality_gaps_flags_empty_simulator_context() -> None:
+    from halios_cli.cli_eval import _eval_quality_gaps
+
+    scenarios = [
+        {
+            "id": "multi-turn-missing-context",
+            "title": "Account check",
+            "goal": "Verify email check",
+            "initial_message": "Hi",
+            "agent_context": {},
+            "simulator_context": {},
+            "arc_messages": ["Provide email when asked"],
+            "max_turns": 4,
+        }
+    ]
+    gaps = _eval_quality_gaps(eval_plan={}, scenarios=scenarios)
+    assert any("simulator_context is empty for a multi-turn scenario" in gap for gap in gaps)

haliosai_cli-2.0.2/halios_cli/_version.py

@@ -1 +0,0 @@
-__version__ = "2.0.2"