fleet-python 0.2.110__tar.gz → 0.2.112__tar.gz

{fleet_python-0.2.110/fleet_python.egg-info → fleet_python-0.2.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.110
+Version: 0.2.112
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/__init__.py

@@ -68,12 +68,15 @@ from .tasks import (
 # Import shared types
 from .types import VerifierFunction
 
+# Import judge data classes
+from .judge import Rubric, Criterion, Image, JudgeResult
+
 # Create a module-level env attribute for convenient access
 from . import env
 from . import global_client as _global_client
 from ._async import global_client as _async_global_client
 
-__version__ = "0.2.110"
+__version__ = "0.2.112"
 
 __all__ = [
     # Core classes
@@ -90,6 +93,11 @@ __all__ = [
     # Task models
     "Task",
     "VerifierFunction",
+    # Judge
+    "Rubric",
+    "Criterion",
+    "Image",
+    "JudgeResult",
     # Exceptions
     "FleetError",
     "FleetAPIError",

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/_async/__init__.py

@@ -44,7 +44,7 @@ from ..types import VerifierFunction
 from .. import env
 from . import global_client as _async_global_client
 
-__version__ = "0.2.110"
+__version__ = "0.2.112"
 
 __all__ = [
     # Core classes

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/_async/base.py

@@ -26,7 +26,7 @@ from .exceptions import (
 try:
     from .. import __version__
 except ImportError:
-    __version__ = "0.2.110"
+    __version__ = "0.2.112"
 
 logger = logging.getLogger(__name__)
 

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/_async/client.py

@@ -54,6 +54,7 @@ from .tasks import Task
 
 if TYPE_CHECKING:
     from .verifiers import AsyncVerifierFunction
+    from .judge import AsyncJudge
 
 
 def _json_default(x: Any) -> Any:
@@ -344,6 +345,7 @@ class AsyncEnv(EnvironmentBase):
         self._client = client
         self._apps: Dict[str, AsyncInstanceClient] = {}
         self._instance: Optional[AsyncInstanceClient] = None
+        self._judge: Optional["AsyncJudge"] = None
 
     @property
     def instance(self) -> AsyncInstanceClient:
@@ -419,6 +421,18 @@ class AsyncEnv(EnvironmentBase):
         mcp_url = f"{self.urls.root}mcp"
         return AsyncMCPResource(url=mcp_url, env_key=self.env_key)
 
+    @property
+    def judge(self) -> "AsyncJudge":
+        """LLM-as-judge grading via orchestrator API."""
+        if self._judge is None:
+            from .judge import AsyncJudge
+
+            self._judge = AsyncJudge(
+                client=self._load_client,
+                instance_id=self.instance_id,
+            )
+        return self._judge
+
     def state(self, uri: str) -> Resource:
         return self.instance.state(uri)
 

fleet_python-0.2.112/fleet/_async/judge.py

@@ -0,0 +1,96 @@
+"""Fleet SDK Judge - Async version.
+
+Provides env.judge.grade() for async verifier scripts.
+"""
+
+from typing import Dict, List, Optional, Union, TYPE_CHECKING
+
+# Import shared classes and helpers from the sync module
+from ..judge import (
+    Criterion,
+    Image,
+    JudgeResult,
+    Rubric,
+    _build_grade_request,
+    _parse_grade_response,
+)
+
+if TYPE_CHECKING:
+    from .base import AsyncWrapper
+
+# Re-export data classes so `from fleet._async.judge import ...` works
+__all__ = [
+    "AsyncJudge",
+    "Criterion",
+    "Image",
+    "JudgeResult",
+    "Rubric",
+]
+
+
+class AsyncJudge:
+    """LLM-as-judge grading — calls orchestrator API, not environment API.
+
+    Accessed as env.judge on AsyncEnv instances.
+    """
+
+    def __init__(self, client: "AsyncWrapper", instance_id: str):
+        self._client = client
+        self._instance_id = instance_id
+
+    async def grade(
+        self,
+        rubric: Union[str, Rubric],
+        submission: Optional[str] = None,
+        *,
+        ground_truth: Optional[Union[str, dict]] = None,
+        problem: Optional[str] = None,
+        context: Optional[str] = None,
+        reference_claims: Optional[str] = None,
+        conversation: Optional[List[dict]] = None,
+        images: Optional[Dict[str, Image]] = None,
+        model: Optional[str] = None,
+        provider: Optional[str] = None,
+        agentic: bool = False,
+        collect: Optional[Dict[str, List[str]]] = None,
+        task_id: Optional[str] = None,
+    ) -> JudgeResult:
+        """Grade a submission using LLM-as-judge via the orchestrator API.
+
+        Returns a JudgeResult (float subclass with .details, .criteria, .feedback)
+        that can be returned directly from a verifier function.
+
+        Args:
+            rubric: Grading rubric — either a string or a structured Rubric object.
+            submission: The agent's final answer / submission text.
+            ground_truth: Expected answer (string or dict).
+            problem: The original problem statement.
+            context: Additional context for the judge.
+            reference_claims: Reference analysis claims.
+            conversation: Conversation history as list of message dicts.
+            images: Named images for the judge (e.g., gold reference, agent output).
+            model: Override LLM model (server picks default if None).
+            provider: Override LLM provider (server picks default if None).
+            agentic: If True, the orchestrator collects artifacts from the instance.
+            collect: File patterns for orchestrator to collect (agentic mode).
+            task_id: Optional task ID for tracking.
+        """
+        body = _build_grade_request(
+            self._instance_id,
+            rubric,
+            submission,
+            ground_truth=ground_truth,
+            problem=problem,
+            context=context,
+            reference_claims=reference_claims,
+            conversation=conversation,
+            images=images,
+            model=model,
+            provider=provider,
+            agentic=agentic,
+            collect=collect,
+            task_id=task_id,
+        )
+
+        response = await self._client.request("POST", "/v1/judge/grade", json=body)
+        return _parse_grade_response(response.json())

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/_async/resources/filesystem.py

@@ -301,6 +301,14 @@ class AsyncFilesystemResource(Resource):
         response = await self.client.request(
             "POST", "/fs/file", json=request.model_dump()
         )
+        if response.status_code == 404:
+            return FileStateResponse(
+                success=True, path=path, exists=False,
+                message=response.json().get("detail", "File not found"),
+            )
+        if response.status_code >= 400:
+            detail = response.json().get("detail", response.text)
+            raise RuntimeError(f"Failed to get file state for '{path}': {detail}")
         return FileStateResponse(**response.json())
 
     async def file_text(self, path: str, max_content_size: int = 102400) -> str:

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/base.py

@@ -27,7 +27,7 @@ from .exceptions import (
 try:
     from . import __version__
 except ImportError:
-    __version__ = "0.2.110"
+    __version__ = "0.2.112"
 
 logger = logging.getLogger(__name__)
 

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/client.py

@@ -59,6 +59,7 @@ from .tasks import Task
 
 if TYPE_CHECKING:
     from .verifiers import SyncVerifierFunction
+    from .judge import SyncJudge
 
 
 def _json_default(x: Any) -> Any:
@@ -348,6 +349,7 @@ class SyncEnv(EnvironmentBase):
         self._client = client
         self._apps: Dict[str, InstanceClient] = {}
         self._instance: Optional[InstanceClient] = None
+        self._judge: Optional["SyncJudge"] = None
         self._manager_url_override: Optional[str] = None  # For URL mode
 
     @property
@@ -431,6 +433,18 @@ class SyncEnv(EnvironmentBase):
         mcp_url = f"{self.urls.root}mcp"
         return SyncMCPResource(url=mcp_url, env_key=self.env_key)
 
+    @property
+    def judge(self) -> "SyncJudge":
+        """LLM-as-judge grading via orchestrator API."""
+        if self._judge is None:
+            from .judge import SyncJudge
+
+            self._judge = SyncJudge(
+                client=self._load_client,
+                instance_id=self.instance_id,
+            )
+        return self._judge
+
     def state(self, uri: str) -> Resource:
         return self.instance.state(uri)
 

fleet_python-0.2.112/fleet/judge.py

@@ -0,0 +1,521 @@
+"""Fleet SDK Judge - LLM-as-Judge grading via orchestrator API.
+
+Provides env.judge.grade() for verifier scripts to grade submissions
+using LLM judges without managing API keys, HTTP calls, or response parsing.
+
+All LLM calls happen server-side on the orchestrator — the SDK just sends
+the rubric, submission, and artifacts, and gets back a score.
+"""
+
+import base64
+import json
+import logging
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .base import SyncWrapper
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes (used by both sync and async)
+# ---------------------------------------------------------------------------
+
+
+def _guess_media_type(filename: str) -> str:
+    """Guess media type from filename extension."""
+    ext = filename.lower().rsplit(".", 1)[-1] if "." in filename else ""
+    return {
+        "png": "image/png",
+        "jpg": "image/jpeg",
+        "jpeg": "image/jpeg",
+        "gif": "image/gif",
+        "webp": "image/webp",
+        "svg": "image/svg+xml",
+    }.get(ext, "image/png")
+
+
+@dataclass
+class Criterion:
+    """A single rubric criterion for grading.
+
+    Args:
+        name: Name of this criterion (e.g., "Takeaway Alignment")
+        max: Maximum points for this criterion
+        levels: Optional mapping of score -> description for each level.
+            Rendered into the description string for the API.
+        description: Optional freeform description (alternative to levels)
+    """
+
+    name: str
+    max: int
+    levels: Optional[Dict[int, str]] = None
+    description: Optional[str] = None
+
+    def _render_description(self) -> str:
+        """Render levels dict + description into a single description string."""
+        parts = []
+        if self.levels:
+            for score in sorted(self.levels.keys(), reverse=True):
+                parts.append(f"- {score} points: {self.levels[score]}")
+        if self.description:
+            parts.append(self.description)
+        return "\n".join(parts) if parts else self.name
+
+    def serialize(self) -> dict:
+        return {
+            "name": self.name,
+            "max_score": self.max,
+            "description": self._render_description(),
+        }
+
+
+@dataclass
+class Rubric:
+    """Structured grading rubric.
+
+    Args:
+        criteria: List of Criterion objects
+        system_prompt: Optional override for the judge system prompt
+    """
+
+    criteria: List[Criterion] = field(default_factory=list)
+    system_prompt: Optional[str] = None
+
+    @property
+    def max_score(self) -> int:
+        return sum(c.max for c in self.criteria)
+
+    def serialize(self) -> dict:
+        d: dict = {
+            "type": "structured",
+            "criteria": [c.serialize() for c in self.criteria],
+        }
+        if self.system_prompt is not None:
+            d["system_prompt"] = self.system_prompt
+        return d
+
+
+class Image:
+    """Reference to an image for LLM judge grading.
+
+    Use the static constructors to create instances:
+        Image.s3("s3://bucket/key")           - S3 URL, fetched server-side
+        Image.from_url("https://...")          - HTTP URL, fetched server-side
+        Image.from_base64(data, "file.png")    - Inline base64 data
+        Image.from_env(env, "plot.png")        - Collect from environment
+    """
+
+    def __init__(
+        self,
+        *,
+        source: str,
+        url: Optional[str] = None,
+        data: Optional[str] = None,
+        filename: Optional[str] = None,
+        media_type: Optional[str] = None,
+        _env: Optional[Any] = None,
+    ):
+        self.source = source
+        self.url = url
+        self.data = data
+        self.filename = filename
+        self.media_type = media_type
+        self._env = _env
+
+    @staticmethod
+    def s3(url: str, media_type: Optional[str] = None) -> "Image":
+        """Reference an image in S3. The orchestrator fetches it server-side."""
+        return Image(source="s3", url=url, media_type=media_type)
+
+    @staticmethod
+    def from_url(url: str, media_type: Optional[str] = None) -> "Image":
+        """Reference an image by HTTP URL. The orchestrator fetches it server-side."""
+        return Image(source="url", url=url, media_type=media_type)
+
+    @staticmethod
+    def from_base64(
+        data: str, filename: str = "image.png", media_type: Optional[str] = None
+    ) -> "Image":
+        """Inline base64 image data."""
+        return Image(
+            source="base64",
+            data=data,
+            filename=filename,
+            media_type=media_type or _guess_media_type(filename),
+        )
+
+    @staticmethod
+    def from_env(env: Any, filename: str) -> "Image":
+        """Collect an image from the environment.
+
+        In non-agentic mode, the SDK collects the image client-side (DB -> notebook -> filesystem)
+        and sends base64 to the orchestrator.
+
+        In agentic mode, only the filename hint is sent and the orchestrator collects it.
+        """
+        return Image(source="env", filename=filename, _env=env)
+
+    def serialize(self, *, label: Optional[str] = None, agentic: bool = False) -> dict:
+        """Serialize for the orchestrator API request body."""
+        d: dict
+        if self.source == "s3":
+            d = {"source": "s3", "url": self.url}
+            if self.media_type:
+                d["media_type"] = self.media_type
+        elif self.source == "url":
+            d = {"source": "url", "url": self.url}
+            if self.media_type:
+                d["media_type"] = self.media_type
+        elif self.source == "base64":
+            d = {
+                "source": "base64",
+                "data": self.data,
+                "media_type": self.media_type or _guess_media_type(self.filename or "image.png"),
+            }
+        elif self.source == "env":
+            if agentic:
+                d = {"source": "collect", "selector": self.filename}
+            else:
+                b64 = _collect_image_from_env(self._env, self.filename)
+                if b64 is None:
+                    d = {"source": "collect", "selector": self.filename}
+                else:
+                    d = {
+                        "source": "base64",
+                        "data": b64,
+                        "media_type": _guess_media_type(self.filename or "image.png"),
+                    }
+        else:
+            raise ValueError(f"Unknown image source: {self.source}")
+
+        if label is not None:
+            d["label"] = label
+        return d
+
+
+class JudgeResult(float):
+    """Float subclass that carries grading details.
+
+    Can be returned directly from a verifier function (it IS a float),
+    but also carries structured metadata from the judge response.
+    """
+
+    def __new__(cls, score: float, *, details: Optional[dict] = None):
+        instance = super().__new__(cls, score)
+        instance.details = details or {}  # type: ignore[attr-defined]
+        instance.criteria = instance.details.get("criteria", [])  # type: ignore[attr-defined]
+        instance.feedback = instance.details.get("feedback", "")  # type: ignore[attr-defined]
+        instance.execution_id = instance.details.get("execution_id", "")  # type: ignore[attr-defined]
+        return instance
+
+
+# ---------------------------------------------------------------------------
+# Image collection helpers
+# ---------------------------------------------------------------------------
+
+
+def _extract_query_rows(result: Any) -> List[Dict[str, Any]]:
+    """Extract rows from a query response, handling various formats."""
+    if result is None:
+        return []
+    # QueryResponse with columns/rows
+    cols = getattr(result, "columns", None)
+    rows = getattr(result, "rows", None)
+    if isinstance(cols, list) and isinstance(rows, list):
+        return [
+            {str(cols[i]): row[i] for i in range(min(len(cols), len(row)))}
+            if isinstance(row, (list, tuple))
+            else row
+            for row in rows
+            if isinstance(row, (list, tuple, dict))
+        ]
+    # Dict with columns/rows
+    if isinstance(result, dict):
+        cols = result.get("columns")
+        rows = result.get("rows")
+        if isinstance(cols, list) and isinstance(rows, list):
+            return [
+                {str(cols[i]): row[i] for i in range(min(len(cols), len(row)))}
+                if isinstance(row, (list, tuple))
+                else row
+                for row in rows
+                if isinstance(row, (list, tuple, dict))
+            ]
+    # Plain list of dicts
+    if isinstance(result, list):
+        return [row for row in result if isinstance(row, dict)]
+    return []
+
+
+def _collect_image_from_env(env: Any, filename: str) -> Optional[str]:
+    """Collect an image from the environment using DB -> notebook -> filesystem strategies.
+
+    Returns base64-encoded image data, or None if not found.
+    """
+    # Strategy 1: DB files table
+    try:
+        current = env.db("current")
+        where = f"path = '{filename}' OR path LIKE '%/{filename}'"
+        rows = _extract_query_rows(
+            current.query(f"SELECT path, hex(content) AS content_hex FROM files WHERE {where}")
+        )
+        candidates = {}
+        for row in rows:
+            path, chex = row.get("path", ""), row.get("content_hex", "")
+            if path and chex:
+                try:
+                    candidates[path] = bytes.fromhex(chex)
+                except Exception:
+                    pass
+        # Prefer non-dataroom paths
+        non_dr = [p for p in candidates if not p.startswith("dataroom/")]
+        best = sorted(non_dr or list(candidates.keys()), key=len)
+        if best:
+            logger.debug("Loaded image from DB: %s", best[0])
+            return base64.b64encode(candidates[best[0]]).decode()
+    except Exception as e:
+        logger.debug("DB image query failed: %s", e)
+
+    # Strategy 2: Notebook cell outputs
+    try:
+        current = env.db("current")
+        nb_rows = _extract_query_rows(
+            current.query(
+                "SELECT path, hex(content) AS content_hex FROM files "
+                "WHERE path LIKE 'notebooks/%.ipynb'"
+            )
+        )
+        for nb_row in nb_rows:
+            chex = nb_row.get("content_hex", "")
+            if not chex:
+                continue
+            try:
+                nb_bytes = bytes.fromhex(chex)
+                nb = json.loads(nb_bytes.decode("utf-8"))
+                for cell in reversed(nb.get("cells", [])):
+                    for output in cell.get("outputs", []):
+                        if output.get("output_type") in ("display_data", "execute_result"):
+                            img_data = output.get("data", {}).get("image/png")
+                            if img_data:
+                                if isinstance(img_data, list):
+                                    img_data = "".join(img_data)
+                                img_data = img_data.strip()
+                                if img_data:
+                                    logger.debug("Loaded image from notebook: %s", nb_row.get("path"))
+                                    return img_data
+            except Exception:
+                pass
+    except Exception as e:
+        logger.debug("Notebook image query failed: %s", e)
+
+    # Strategy 3: Filesystem fallback
+    search_paths = [
+        filename,
+        f"/app/workspace/{filename}",
+        f"/workspace/{filename}",
+    ]
+    for fp in search_paths:
+        try:
+            if os.path.exists(fp):
+                with open(fp, "rb") as f:
+                    logger.debug("Loaded image from filesystem: %s", fp)
+                    return base64.b64encode(f.read()).decode()
+        except Exception:
+            pass
+
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Accumulator printing (verifier protocol)
+# ---------------------------------------------------------------------------
+
+
+def _print_accumulators(data: dict) -> None:
+    """Print error/success accumulators from orchestrator response (verifier protocol)."""
+    acc = data.get("accumulators")
+    if not acc:
+        return
+
+    errors = acc.get("errors")
+    if errors:
+        print("[STDOUT] >>> ERROR_ACCUMULATOR >>>")
+        print(json.dumps(errors))
+        print("<<< ERROR_ACCUMULATOR <<<")
+
+    successes = acc.get("successes")
+    if successes:
+        print(">>> SUCCESS_ACCUMULATOR >>>")
+        print(json.dumps(successes))
+        print("<<< SUCCESS_ACCUMULATOR <<<")
+
+    grading_details = acc.get("grading_details")
+    if grading_details:
+        print(">>> GRADING_DETAILS >>>")
+        print(json.dumps(grading_details))
+        print("<<< GRADING_DETAILS <<<")
+
+    timing = acc.get("timing")
+    if timing:
+        print(
+            f">>> TIMING: started={timing.get('started_ms')}, "
+            f"finished={timing.get('finished_ms')}, "
+            f"duration={timing.get('duration_ms')}ms <<<"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Request body builder (shared by sync and async)
+# ---------------------------------------------------------------------------
+
+
+def _build_grade_request(
+    instance_id: str,
+    rubric: Union[str, Rubric],
+    submission: Optional[str],
+    *,
+    ground_truth: Optional[Union[str, dict]] = None,
+    problem: Optional[str] = None,
+    context: Optional[str] = None,
+    reference_claims: Optional[str] = None,
+    conversation: Optional[List[dict]] = None,
+    images: Optional[Dict[str, Image]] = None,
+    model: Optional[str] = None,
+    provider: Optional[str] = None,
+    agentic: bool = False,
+    collect: Optional[Dict[str, List[str]]] = None,
+    task_id: Optional[str] = None,
+) -> dict:
+    """Build the JSON request body for POST /v1/judge/grade."""
+    body: Dict[str, Any] = {
+        "instance_id": instance_id,
+        "submission": submission,
+        "agentic": agentic,
+    }
+
+    # Rubric
+    if isinstance(rubric, str):
+        body["rubric"] = {"type": "string", "text": rubric}
+    elif isinstance(rubric, Rubric):
+        body["rubric"] = rubric.serialize()
+    else:
+        raise TypeError(f"rubric must be str or Rubric, got {type(rubric)}")
+
+    # Optional fields
+    if ground_truth is not None:
+        body["ground_truth"] = ground_truth
+    if problem is not None:
+        body["problem"] = problem
+    if reference_claims is not None:
+        # Fold reference_claims into context
+        if context:
+            context = f"{context}\n\n## Reference Claims\n{reference_claims}"
+        else:
+            context = f"## Reference Claims\n{reference_claims}"
+    if context is not None:
+        body["context"] = context
+    if conversation is not None:
+        body["conversation"] = [
+            {"role": m["role"], "content": m["content"]} for m in conversation
+        ]
+    if model is not None:
+        body["model"] = model
+    if provider is not None:
+        body["provider"] = provider
+    if task_id is not None:
+        body["task_id"] = task_id
+    if collect is not None:
+        body["collect"] = collect
+
+    # Serialize images as labeled array
+    if images:
+        body["images"] = [
+            img.serialize(label=label, agentic=agentic)
+            for label, img in images.items()
+        ]
+
+    return body
+
+
+def _parse_grade_response(data: dict) -> JudgeResult:
+    """Parse orchestrator response into JudgeResult and print accumulators."""
+    _print_accumulators(data)
+    score = float(data.get("normalized_score", 0.0))
+    return JudgeResult(score, details=data)
+
+
+# ---------------------------------------------------------------------------
+# Sync judge
+# ---------------------------------------------------------------------------
+
+
+class SyncJudge:
+    """LLM-as-judge grading — calls orchestrator API, not environment API.
+
+    Accessed as env.judge on SyncEnv instances.
+    """
+
+    def __init__(self, client: "SyncWrapper", instance_id: str):
+        self._client = client
+        self._instance_id = instance_id
+
+    def grade(
+        self,
+        rubric: Union[str, Rubric],
+        submission: Optional[str] = None,
+        *,
+        ground_truth: Optional[Union[str, dict]] = None,
+        problem: Optional[str] = None,
+        context: Optional[str] = None,
+        reference_claims: Optional[str] = None,
+        conversation: Optional[List[dict]] = None,
+        images: Optional[Dict[str, Image]] = None,
+        model: Optional[str] = None,
+        provider: Optional[str] = None,
+        agentic: bool = False,
+        collect: Optional[Dict[str, List[str]]] = None,
+        task_id: Optional[str] = None,
+    ) -> JudgeResult:
+        """Grade a submission using LLM-as-judge via the orchestrator API.
+
+        Returns a JudgeResult (float subclass with .details, .criteria, .feedback)
+        that can be returned directly from a verifier function.
+
+        Args:
+            rubric: Grading rubric — either a string or a structured Rubric object.
+            submission: The agent's final answer / submission text.
+            ground_truth: Expected answer (string or dict).
+            problem: The original problem statement.
+            context: Additional context for the judge.
+            reference_claims: Reference analysis claims (folded into context).
+            conversation: Conversation history as list of message dicts.
+            images: List of Image objects for the judge.
+            model: Override LLM model (server picks default if None).
+            provider: Override LLM provider (server picks default if None).
+            agentic: If True, the orchestrator collects artifacts from the instance.
+            collect: File patterns for orchestrator to collect (agentic mode).
+            task_id: Optional task ID for tracking.
+        """
+        body = _build_grade_request(
+            self._instance_id,
+            rubric,
+            submission,
+            ground_truth=ground_truth,
+            problem=problem,
+            context=context,
+            reference_claims=reference_claims,
+            conversation=conversation,
+            images=images,
+            model=model,
+            provider=provider,
+            agentic=agentic,
+            collect=collect,
+            task_id=task_id,
+        )
+
+        response = self._client.request("POST", "/v1/judge/grade", json=body)
+        return _parse_grade_response(response.json())

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet/resources/filesystem.py

@@ -301,6 +301,14 @@ class FilesystemResource(Resource):
         response = self.client.request(
             "POST", "/fs/file", json=request.model_dump()
         )
+        if response.status_code == 404:
+            return FileStateResponse(
+                success=True, path=path, exists=False,
+                message=response.json().get("detail", "File not found"),
+            )
+        if response.status_code >= 400:
+            detail = response.json().get("detail", response.text)
+            raise RuntimeError(f"Failed to get file state for '{path}': {detail}")
         return FileStateResponse(**response.json())
 
     def file_text(self, path: str, max_content_size: int = 102400) -> str:

{fleet_python-0.2.110 → fleet_python-0.2.112/fleet_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.110
+Version: 0.2.112
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.110 → fleet_python-0.2.112}/fleet_python.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ fleet/client.py
 fleet/config.py
 fleet/exceptions.py
 fleet/global_client.py
+fleet/judge.py
 fleet/models.py
 fleet/tasks.py
 fleet/types.py
@@ -42,6 +43,7 @@ fleet/_async/base.py
 fleet/_async/client.py
 fleet/_async/exceptions.py
 fleet/_async/global_client.py
+fleet/_async/judge.py
 fleet/_async/models.py
 fleet/_async/tasks.py
 fleet/_async/env/__init__.py

{fleet_python-0.2.110 → fleet_python-0.2.112}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "fleet-python"
 
-version = "0.2.110"
+version = "0.2.112"
 description = "Python SDK for Fleet environments"
 authors = [
     {name = "Fleet AI", email = "nic@fleet.so"},