fleet-python 0.2.115__tar.gz → 0.2.116__tar.gz

{fleet_python-0.2.115/fleet_python.egg-info → fleet_python-0.2.116}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.115
+Version: 0.2.116
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/__init__.py

@@ -76,7 +76,7 @@ from . import env
 from . import global_client as _global_client
 from ._async import global_client as _async_global_client
 
-__version__ = "0.2.115"
+__version__ = "0.2.116"
 
 __all__ = [
     # Core classes

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/_async/__init__.py

@@ -44,7 +44,7 @@ from ..types import VerifierFunction
 from .. import env
 from . import global_client as _async_global_client
 
-__version__ = "0.2.115"
+__version__ = "0.2.116"
 
 __all__ = [
     # Core classes

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/_async/base.py

@@ -26,7 +26,7 @@ from .exceptions import (
 try:
     from .. import __version__
 except ImportError:
-    __version__ = "0.2.115"
+    __version__ = "0.2.116"
 
 logger = logging.getLogger(__name__)
 

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/_async/client.py

@@ -601,7 +601,6 @@ class AsyncFleet:
         )
 
         instance = AsyncEnv(client=self.client, **response.json())
-        await instance.instance.load()
         return instance
 
     async def make_for_task(self, task: Task) -> AsyncEnv:
@@ -653,7 +652,6 @@ class AsyncFleet:
         else:
             response = await self.client.request("GET", f"/v1/env/instances/{instance_id}")
             instance = AsyncEnv(client=self.client, **response.json())
-            await instance.instance.load()
             return instance
 
     def _create_url_instance(self, base_url: str) -> AsyncEnv:

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/base.py

@@ -27,7 +27,7 @@ from .exceptions import (
 try:
     from . import __version__
 except ImportError:
-    __version__ = "0.2.115"
+    __version__ = "0.2.116"
 
 logger = logging.getLogger(__name__)
 

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/client.py

@@ -613,7 +613,6 @@ class Fleet:
         )
 
         instance = SyncEnv(client=self.client, **response.json())
-        instance.instance.load()
         return instance
 
     def make_for_task(self, task: Task) -> SyncEnv:
@@ -665,7 +664,6 @@ class Fleet:
         else:
             response = self.client.request("GET", f"/v1/env/instances/{instance_id}")
             instance = SyncEnv(client=self.client, **response.json())
-            instance.instance.load()
             return instance
 
     def _create_url_instance(self, base_url: str) -> SyncEnv:

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet/judge.py

@@ -823,6 +823,54 @@ def _parse_grade_response(data: dict) -> JudgeResult:
     return JudgeResult(score, details=data)
 
 
+def _print_criteria_markers(criteria: list) -> None:
+    """Emit ``>>> CRITERIA >>>`` stdout markers for structured criteria display.
+
+    The orchestrator (theseus PR #1967) scans verifier stdout for these
+    markers and wraps the execution result so the client (client PR #1737)
+    can render an expandable rubric breakdown.
+
+    Converts from the orchestrator judge-response format::
+
+        {"name": str, "score": int, "max_score": int, "reasoning": str}
+
+    to the client-expected marker format::
+
+        {"criteria": str, "score": float, "score_out_of": float, "description"?: str}
+
+    Each criterion's score is normalised to a 0.0–1.0 float using its own
+    ``max_score``.
+    """
+    marker_criteria = []
+    for c in criteria:
+        name = c.get("name", "")
+        cscore = c.get("score", 0)
+        cmax = c.get("max_score", 0)
+
+        # Normalise per-criterion score to 0.0–1.0
+        if cmax and float(cmax) > 0:
+            norm_score = float(cscore) / float(cmax)
+        else:
+            norm_score = float(cscore)
+
+        entry: dict = {
+            "criteria": name,
+            "score": round(norm_score, 4),
+            "score_out_of": 1.0,
+        }
+
+        reasoning = c.get("reasoning", "")
+        if reasoning:
+            entry["description"] = reasoning
+
+        marker_criteria.append(entry)
+
+    if marker_criteria:
+        print(">>> CRITERIA >>>")
+        print(json.dumps(marker_criteria))
+        print("<<< CRITERIA <<<")
+
+
 def _print_judge_result(data: dict) -> None:
     """Print detailed judge grading result for verifier stdout capture."""
     model = data.get("model_used", "unknown")
@@ -848,6 +896,12 @@ def _print_judge_result(data: dict) -> None:
             if len(reasoning) > 200:
                 reasoning = reasoning[:200] + "..."
             print(f"[C]   {name}: {cscore}/{cmax} — {reasoning}")
+
+        # Emit structured criteria via stdout markers so the orchestrator
+        # (_extract_criteria_from_stdout) and client can render a rubric
+        # breakdown.  Schema per element:
+        #   {"criteria": str, "score": float, "score_out_of": float, "description"?: str}
+        _print_criteria_markers(criteria)
     else:
         print(f"[C] Score: {normalized:.2f}")
 

{fleet_python-0.2.115 → fleet_python-0.2.116/fleet_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.115
+Version: 0.2.116
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.115 → fleet_python-0.2.116}/fleet_python.egg-info/SOURCES.txt

@@ -117,6 +117,7 @@ tests/test_app_method.py
 tests/test_expect_exactly.py
 tests/test_expect_only.py
 tests/test_instance_dispatch.py
+tests/test_judge_criteria_markers.py
 tests/test_sqlite_resource_dual_mode.py
 tests/test_sqlite_shared_memory_behavior.py
 tests/test_verifier_from_string.py

{fleet_python-0.2.115 → fleet_python-0.2.116}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "fleet-python"
 
-version = "0.2.115"
+version = "0.2.116"
 description = "Python SDK for Fleet environments"
 authors = [
     {name = "Fleet AI", email = "nic@fleet.so"},

fleet_python-0.2.116/tests/test_judge_criteria_markers.py

@@ -0,0 +1,192 @@
+"""Tests for structured criteria stdout markers in fleet.judge.
+
+Validates that _print_criteria_markers emits the correct
+>>> CRITERIA >>> / <<< CRITERIA <<< markers that the orchestrator
+(theseus PR #1967) and client (client PR #1737) expect.
+"""
+
+import json
+import re
+from io import StringIO
+from unittest.mock import patch
+
+import pytest
+
+from fleet.judge import (
+    _print_criteria_markers,
+    _print_judge_result,
+    _parse_grade_response,
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_MARKER_RE = re.compile(
+    r">>> CRITERIA >>>\s*\n(.*?)\n<<< CRITERIA <<<",
+    re.DOTALL,
+)
+
+
+def _capture_print(fn, *args, **kwargs):
+    """Capture all print() output from a function call."""
+    buf = StringIO()
+    with patch("builtins.print", side_effect=lambda *a, **kw: buf.write(" ".join(str(x) for x in a) + "\n")):
+        fn(*args, **kwargs)
+    return buf.getvalue()
+
+
+def _extract_criteria_from_stdout(stdout: str):
+    """Mirror the orchestrator's extraction logic (theseus PR #1967)."""
+    m = _MARKER_RE.search(stdout)
+    if not m:
+        return None
+    parsed = json.loads(m.group(1).strip())
+    if isinstance(parsed, list):
+        return parsed
+    return None
+
+
+# ---------------------------------------------------------------------------
+# _print_criteria_markers
+# ---------------------------------------------------------------------------
+
+class TestPrintCriteriaMarkers:
+    """Tests for _print_criteria_markers."""
+
+    def test_basic_criteria_output(self):
+        """Emits valid markers with normalised scores."""
+        criteria = [
+            {"name": "Accuracy", "score": 8, "max_score": 10, "reasoning": "Good job"},
+            {"name": "Style", "score": 5, "max_score": 5, "reasoning": "Perfect"},
+        ]
+        stdout = _capture_print(_print_criteria_markers, criteria)
+
+        parsed = _extract_criteria_from_stdout(stdout)
+        assert parsed is not None, f"Markers not found in stdout:\n{stdout}"
+        assert len(parsed) == 2
+
+        assert parsed[0]["criteria"] == "Accuracy"
+        assert parsed[0]["score"] == pytest.approx(0.8, abs=0.01)
+        assert parsed[0]["score_out_of"] == 1.0
+        assert parsed[0]["description"] == "Good job"
+
+        assert parsed[1]["criteria"] == "Style"
+        assert parsed[1]["score"] == pytest.approx(1.0, abs=0.01)
+        assert parsed[1]["score_out_of"] == 1.0
+
+    def test_zero_max_score_passthrough(self):
+        """When max_score is 0, raw score passes through."""
+        criteria = [
+            {"name": "Metric", "score": 0.75, "max_score": 0},
+        ]
+        stdout = _capture_print(_print_criteria_markers, criteria)
+        parsed = _extract_criteria_from_stdout(stdout)
+        assert parsed is not None
+        assert parsed[0]["score"] == pytest.approx(0.75, abs=0.01)
+
+    def test_empty_criteria_no_markers(self):
+        """Empty list should produce no markers."""
+        stdout = _capture_print(_print_criteria_markers, [])
+        assert ">>> CRITERIA >>>" not in stdout
+
+    def test_reasoning_maps_to_description(self):
+        """The 'reasoning' field maps to 'description' in the marker schema."""
+        criteria = [
+            {"name": "Test", "score": 3, "max_score": 5, "reasoning": "Some reasoning here"},
+        ]
+        stdout = _capture_print(_print_criteria_markers, criteria)
+        parsed = _extract_criteria_from_stdout(stdout)
+        assert parsed[0]["description"] == "Some reasoning here"
+
+    def test_missing_reasoning_no_description(self):
+        """When reasoning is empty, description key should be absent."""
+        criteria = [
+            {"name": "Test", "score": 3, "max_score": 5, "reasoning": ""},
+        ]
+        stdout = _capture_print(_print_criteria_markers, criteria)
+        parsed = _extract_criteria_from_stdout(stdout)
+        assert "description" not in parsed[0]
+
+    def test_output_parseable_by_orchestrator_regex(self):
+        """Ensure the output matches the exact regex the orchestrator uses."""
+        criteria = [
+            {"name": "A", "score": 1, "max_score": 2, "reasoning": "half"},
+        ]
+        stdout = _capture_print(_print_criteria_markers, criteria)
+
+        # Use the exact regex from theseus PR #1967
+        m = re.search(
+            r">>> CRITERIA >>>\s*\n(.*?)\n<<< CRITERIA <<<",
+            stdout,
+            re.DOTALL,
+        )
+        assert m is not None, "Output doesn't match orchestrator regex"
+        data = json.loads(m.group(1).strip())
+        assert isinstance(data, list)
+        assert data[0]["criteria"] == "A"
+
+
+# ---------------------------------------------------------------------------
+# _print_judge_result integration
+# ---------------------------------------------------------------------------
+
+class TestPrintJudgeResult:
+    """Tests for _print_judge_result emitting criteria markers."""
+
+    def test_criteria_markers_emitted(self):
+        """_print_judge_result emits criteria markers when criteria present."""
+        data = {
+            "model_used": "claude-sonnet",
+            "provider_used": "anthropic",
+            "total_score": 15,
+            "max_score": 20,
+            "normalized_score": 0.75,
+            "criteria": [
+                {"name": "Accuracy", "score": 8, "max_score": 10, "reasoning": "Good"},
+                {"name": "Style", "score": 7, "max_score": 10, "reasoning": "Decent"},
+            ],
+        }
+        stdout = _capture_print(_print_judge_result, data)
+        parsed = _extract_criteria_from_stdout(stdout)
+        assert parsed is not None
+        assert len(parsed) == 2
+
+    def test_no_criteria_no_markers(self):
+        """_print_judge_result doesn't emit markers when no criteria."""
+        data = {
+            "model_used": "claude-sonnet",
+            "provider_used": "anthropic",
+            "total_score": 0,
+            "max_score": 0,
+            "normalized_score": 0.5,
+        }
+        stdout = _capture_print(_print_judge_result, data)
+        assert ">>> CRITERIA >>>" not in stdout
+
+
+# ---------------------------------------------------------------------------
+# _parse_grade_response integration
+# ---------------------------------------------------------------------------
+
+class TestParseGradeResponse:
+    """Tests for _parse_grade_response emitting criteria markers."""
+
+    def test_full_flow_emits_markers(self):
+        """_parse_grade_response → _print_judge_result → criteria markers."""
+        data = {
+            "model_used": "claude-sonnet",
+            "provider_used": "anthropic",
+            "total_score": 9,
+            "max_score": 10,
+            "normalized_score": 0.9,
+            "criteria": [
+                {"name": "Completeness", "score": 9, "max_score": 10, "reasoning": "Almost perfect"},
+            ],
+        }
+        stdout = _capture_print(_parse_grade_response, data)
+        parsed = _extract_criteria_from_stdout(stdout)
+        assert parsed is not None
+        assert parsed[0]["criteria"] == "Completeness"
+        assert parsed[0]["score"] == pytest.approx(0.9, abs=0.01)