eval-protocol 0.3.30__tar.gz → 0.3.32__tar.gz

{eval_protocol-0.3.30/eval_protocol.egg-info → eval_protocol-0.3.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.30
+Version: 0.3.32
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.30 → eval_protocol-0.3.32}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-05-29T16:09:24-0700",
+ "date": "2026-07-06T12:57:04-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "1bd5447a3afbca3b71e0f0d205ed7cff6c3afe5d",
- "version": "0.3.30"
+ "full-revisionid": "2c0867156a720cf218a101dd37e51f4d4861ae00",
+ "version": "0.3.32"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.30 → eval_protocol-0.3.32}/eval_protocol/adapters/fireworks_tracing.py

@@ -15,9 +15,8 @@ from typing import Any, Dict, List, Optional, Protocol
 import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
+from eval_protocol.tracing import PayloadType, decode_payloads
 from .base import BaseAdapter
-from .lp_deserializer import decompress_and_parse_lp
-from .r3_deserializer import decompress_and_parse_r3
 from .utils import extract_messages_from_data
 from ..common_utils import get_user_agent
 
@@ -102,45 +101,42 @@ def convert_trace_dict_to_evaluation_row(
                 ):
                     break  # Break early if we've found all the metadata we need
 
-        # Extract router replay payloads when present
+        # Decoding lives in eval_protocol.tracing; here we only map results onto the row.
         payloads = trace.get("payloads")
         if isinstance(payloads, dict):
-            router_replay = payloads.get("router_replay")
-            if isinstance(router_replay, dict) and router_replay.get("data"):
-                try:
-                    matrices, r3_meta = decompress_and_parse_r3(router_replay["data"])
-                    if execution_metadata.extra is None:
-                        execution_metadata.extra = {}
-                    execution_metadata.extra["routing_matrices"] = matrices
-                    execution_metadata.extra["routing_metadata"] = r3_meta
-                except Exception as e:
-                    logger.warning("Failed to decompress R3 payload for trace %s: %s", trace.get("id"), e)
-
-            logprobs_payload = payloads.get("logprobs")
-            if isinstance(logprobs_payload, dict) and logprobs_payload.get("data"):
-                try:
-                    logprobs, token_ids, lp_meta = decompress_and_parse_lp(logprobs_payload["data"])
-                    if execution_metadata.extra is None:
-                        execution_metadata.extra = {}
-                    execution_metadata.extra["completion_logprobs"] = logprobs
-                    if token_ids is not None:
-                        execution_metadata.extra["completion_token_ids"] = token_ids
-                    execution_metadata.extra["logprobs_metadata"] = lp_meta
-
-                    for i in range(len(messages) - 1, -1, -1):
-                        if messages[i].role == "assistant":
-                            content_entries = [{"logprob": lp} for lp in logprobs]
-                            if token_ids is not None:
-                                for entry, tid in zip(content_entries, token_ids):
-                                    entry["token_id"] = tid
-                            messages[i].logprobs = {"content": content_entries}
-                            break
-                except Exception as e:
-                    logger.warning(
-                        "Failed to decompress logprobs payload for trace %s: %s",
-                        trace.get("id"),
-                        e,
-                    )
+            decoded = decode_payloads(
+                payloads,
+                on_error=lambda pt, e: logger.warning(
+                    "Failed to decode %s payload for trace %s: %s", pt.value, trace.get("id"), e
+                ),
+            )
+            if decoded and execution_metadata.extra is None:
+                execution_metadata.extra = {}
+
+            if (dp := decoded.get(PayloadType.ROUTER_REPLAY)) is not None:
+                execution_metadata.extra["routing_matrices"] = dp.value
+                execution_metadata.extra["routing_metadata"] = dp.metadata
+
+            if (dp := decoded.get(PayloadType.LOGPROBS)) is not None:
+                logprobs = dp.value
+                token_ids = dp.token_ids
+                execution_metadata.extra["completion_logprobs"] = logprobs
+                if token_ids is not None:
+                    execution_metadata.extra["completion_token_ids"] = token_ids
+                execution_metadata.extra["logprobs_metadata"] = dp.metadata
+
+                for i in range(len(messages) - 1, -1, -1):
+                    if messages[i].role == "assistant":
+                        content_entries = [{"logprob": lp} for lp in logprobs]
+                        if token_ids is not None:
+                            for entry, tid in zip(content_entries, token_ids):
+                                entry["token_id"] = tid
+                        messages[i].logprobs = {"content": content_entries}
+                        break
+
+            if (dp := decoded.get(PayloadType.PROMPT_TOKEN_IDS)) is not None:
+                execution_metadata.extra["prompt_token_ids"] = dp.value
+                execution_metadata.extra["prompt_token_ids_metadata"] = dp.metadata
 
         return EvaluationRow(
             messages=messages,

eval_protocol-0.3.32/eval_protocol/adapters/lp_deserializer.py

@@ -0,0 +1,41 @@
+"""Deprecated compatibility shim for ``eval_protocol.tracing.logprobs``.
+
+Import from ``eval_protocol.tracing.logprobs`` (or ``decode_payloads`` from
+``eval_protocol.tracing``) instead. This module re-exports the LP/v1 helpers
+that lived here before the tracing package refactor.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+warnings.warn(
+    "eval_protocol.adapters.lp_deserializer is deprecated; "
+    "import from eval_protocol.tracing.logprobs instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from eval_protocol.tracing.logprobs import (  # noqa: E402
+    ENTRY_FORMAT,
+    ENTRY_SIZE,
+    HEADER_FORMAT,
+    HEADER_SIZE,
+    HEADER_VERSION,
+    MAGIC,
+    MISSING_TOKEN_ID,
+    decompress_and_parse_lp,
+    parse_logprobs,
+)
+
+__all__ = [
+    "ENTRY_FORMAT",
+    "ENTRY_SIZE",
+    "HEADER_FORMAT",
+    "HEADER_SIZE",
+    "HEADER_VERSION",
+    "MAGIC",
+    "MISSING_TOKEN_ID",
+    "decompress_and_parse_lp",
+    "parse_logprobs",
+]

eval_protocol-0.3.32/eval_protocol/adapters/r3_deserializer.py

@@ -0,0 +1,41 @@
+"""Deprecated compatibility shim for ``eval_protocol.tracing.router_replay``.
+
+Import from ``eval_protocol.tracing.router_replay`` (or ``decode_payloads`` from
+``eval_protocol.tracing``) instead. This module re-exports the R3/v1 helpers
+that lived here before the tracing package refactor.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+warnings.warn(
+    "eval_protocol.adapters.r3_deserializer is deprecated; "
+    "import from eval_protocol.tracing.router_replay instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from eval_protocol.tracing.router_replay import (  # noqa: E402
+    BITS_PER_BYTE,
+    HEADER_FORMAT,
+    HEADER_SIZE,
+    MAGIC,
+    _RoutingDtype,
+    _SelectorMode,
+    _parse_header,
+    _read_bitmap_positions,
+    decompress_and_parse_r3,
+)
+
+__all__ = [
+    "BITS_PER_BYTE",
+    "HEADER_FORMAT",
+    "HEADER_SIZE",
+    "MAGIC",
+    "_RoutingDtype",
+    "_SelectorMode",
+    "_parse_header",
+    "_read_bitmap_positions",
+    "decompress_and_parse_r3",
+]

{eval_protocol-0.3.30 → eval_protocol-0.3.32}/eval_protocol/integrations/fireworks_v1_completions_client.py

@@ -393,35 +393,50 @@ class FireworksV1CompletionsClient:
         finish_reason = str(choice.get("finish_reason") or "unknown")
 
         raw_output = choice.get("raw_output") if isinstance(choice.get("raw_output"), dict) else {}
-        completion_token_ids = _normalize_token_id_sequence(
-            choice.get("token_ids") or raw_output.get("completion_token_ids") or []
-        )
         choice_prompt_token_ids = _normalize_token_id_sequence(
             choice.get("prompt_token_ids") or raw_output.get("prompt_token_ids") or normalized_prompt_token_ids
         )
 
+        # -- Extract per-token ids and logprobs together --------------------
+        # Both come from the same ``content[]`` array entry-by-entry, so they are
+        # inherently the same length and aligned. Reading ids from a different
+        # source (top-level token_ids) and re-encoding decoded text when it is
+        # absent drops the trailing end-of-turn token and misaligns per-token
+        # logprobs, corrupting inference KLD — so we never do that.
+        choice_logprobs = choice.get("logprobs")
+        content_entries = choice_logprobs.get("content") if isinstance(choice_logprobs, dict) else None
+        if not content_entries:
+            raise RuntimeError(
+                "Fireworks /v1/completions returned no content[] logprobs entries. "
+                "This client requires the boolean logprobs=True (content) shape to "
+                "recover exact per-token ids and sampling logprobs. Refusing to "
+                "re-encode decoded text: retokenization drops the end-of-turn token "
+                "and misaligns per-token logprobs, corrupting inference KLD. "
+                f"choice keys={list(choice.keys())}"
+            )
+
+        completion_token_ids: List[int] = []
+        completion_logprobs: List[float] = []
+        for index, entry in enumerate(content_entries):
+            if not isinstance(entry, dict) or entry.get("token_id") is None:
+                raise RuntimeError(
+                    "Fireworks /v1/completions content[] entry is missing token_id "
+                    f"at index {index}; cannot align per-token logprobs to token ids "
+                    "without re-encoding. Refusing to return corrupted data."
+                )
+            if entry.get("sampling_logprob") is None:
+                raise RuntimeError(
+                    "Fireworks /v1/completions content[] entry is missing "
+                    f"sampling_logprob at index {index}. The sampling logprob is the "
+                    "exact value the sampler drew with and is required for correct "
+                    "inference KLD; refusing to substitute the rounded logprob or 0.0."
+                )
+            completion_token_ids.append(int(entry["token_id"]))
+            completion_logprobs.append(float(entry["sampling_logprob"]))
+
         completion_text = self.decode_token_ids(token_ids=completion_token_ids)
         if not completion_text:
             completion_text = str(choice.get("text") or "")
-        if not completion_token_ids and completion_text:
-            tokenizer = self._get_tokenizer()
-            completion_token_ids = list(tokenizer.encode(completion_text, add_special_tokens=False))
-
-        # -- Extract logprobs -----------------------------------------------
-        completion_logprobs: List[float] = []
-        choice_logprobs = choice.get("logprobs")
-        if isinstance(choice_logprobs, dict):
-            token_logprobs = choice_logprobs.get("token_logprobs") or []
-            if token_logprobs:
-                completion_logprobs = [float(lp) if lp is not None else 0.0 for lp in token_logprobs]
-            else:
-                content_logprobs = choice_logprobs.get("content") or []
-                completion_logprobs = [
-                    float(entry.get("logprob", 0.0)) if isinstance(entry, dict) else 0.0
-                    for entry in content_logprobs
-                ]
-        elif isinstance(choice_logprobs, list):
-            completion_logprobs = [float(lp) if lp is not None else 0.0 for lp in choice_logprobs]
 
         # -- Build message via parser or raw --------------------------------
         if self.tool_call_parser is not None:

{eval_protocol-0.3.30 → eval_protocol-0.3.32}/eval_protocol/pytest/tracing_utils.py

@@ -63,6 +63,8 @@ def _merge_payloads_into_longest_row(longest_row: EvaluationRow, rows: List[Eval
             for key in (
                 "completion_logprobs",
                 "completion_token_ids",
+                "prompt_token_ids",
+                "prompt_token_ids_metadata",
                 "logprobs_metadata",
                 "routing_matrices",
                 "routing_metadata",

eval_protocol-0.3.32/eval_protocol/tracing/__init__.py

@@ -0,0 +1,31 @@
+"""Decode Fireworks tracing-gateway payloads.
+
+Standalone, dependency-light helpers (stdlib + ``zstandard`` only) for turning
+the binary/JSON ``payloads`` returned by the Fireworks tracing gateway
+(``GET /traces?include_payloads=true``) into Python values. No EvaluationRow or
+rollout machinery required -- usable on its own.
+
+Typical use::
+
+    from eval_protocol.tracing import decode_payloads, PayloadType
+
+    decoded = decode_payloads(trace["payloads"])
+    decoded[PayloadType.PROMPT_TOKEN_IDS].value   # List[int]
+    decoded[PayloadType.LOGPROBS].value           # List[float]
+    decoded[PayloadType.ROUTER_REPLAY].value      # List[Optional[str]]
+
+See ``README.md`` in this package for details.
+"""
+
+from __future__ import annotations
+
+from .registry import decode_payload, decode_payloads, decode_trace
+from .types import DecodedPayload, PayloadType
+
+__all__ = [
+    "PayloadType",
+    "DecodedPayload",
+    "decode_payloads",
+    "decode_payload",
+    "decode_trace",
+]

eval_protocol-0.3.30/eval_protocol/adapters/lp_deserializer.py → eval_protocol-0.3.32/eval_protocol/tracing/logprobs.py

@@ -12,6 +12,8 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import zstandard as zstd
 
+from .types import DecodedPayload, PayloadType
+
 MAGIC = b"LP01"
 HEADER_VERSION = 1
 MISSING_TOKEN_ID = -1
@@ -107,3 +109,18 @@ def decompress_and_parse_lp(data_b64: str) -> Tuple[List[float], Optional[List[i
     decompressor = zstd.ZstdDecompressor()
     raw = decompressor.decompress(compressed)
     return parse_logprobs(raw)
+
+
+def decode_logprobs(data_b64: str) -> DecodedPayload:
+    """Decode a gateway ``payloads.logprobs.data`` blob into a ``DecodedPayload``.
+
+    ``value`` is the per-completion-token logprob list; per-token ids (when all
+    valid) are available under ``token_ids``.
+    """
+    logprobs, token_ids, metadata = decompress_and_parse_lp(data_b64)
+    return DecodedPayload(
+        payload_type=PayloadType.LOGPROBS,
+        value=logprobs,
+        metadata=metadata,
+        token_ids=token_ids,
+    )

eval_protocol-0.3.32/eval_protocol/tracing/prompt_token_ids.py

@@ -0,0 +1,34 @@
+"""``pti/v1`` decoder for prompt token ID payloads.
+
+Inverse of the tracing gateway's ``serialize_prompt_token_ids``: the gateway
+stores prompt token IDs as ``base64(zstd(json.dumps(token_ids)))`` -- a compact
+JSON int array, no bespoke binary header.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Any, Dict, List, Tuple
+
+import zstandard as zstd
+
+from .types import DecodedPayload, PayloadType
+
+
+def parse_prompt_token_ids(raw: bytes) -> Tuple[List[int], Dict[str, Any]]:
+    """Parse uncompressed ``pti/v1`` bytes (a JSON int array) into ids + metadata."""
+    token_ids = json.loads(raw)
+    metadata: Dict[str, Any] = {"scope": "prompt_only", "token_count": len(token_ids)}
+    return token_ids, metadata
+
+
+def decode_prompt_token_ids(data_b64: str) -> DecodedPayload:
+    """Decode a gateway ``payloads.prompt_token_ids.data`` blob."""
+    raw = zstd.ZstdDecompressor().decompress(base64.b64decode(data_b64))
+    token_ids, metadata = parse_prompt_token_ids(raw)
+    return DecodedPayload(
+        payload_type=PayloadType.PROMPT_TOKEN_IDS,
+        value=token_ids,
+        metadata=metadata,
+    )

eval_protocol-0.3.32/eval_protocol/tracing/registry.py

@@ -0,0 +1,88 @@
+"""Decoder registry + master decode for tracing-gateway payloads.
+
+Adding a new payload type is a single entry in ``PAYLOAD_DECODERS`` (plus its
+decoder module). Callers use the master :func:`decode_payloads` /
+:func:`decode_trace` and never stitch per-type decoders together.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable, Dict, Optional
+
+from .logprobs import decode_logprobs
+from .prompt_token_ids import decode_prompt_token_ids
+from .router_replay import decode_router_replay
+from .types import DecodedPayload, PayloadType
+
+logger = logging.getLogger(__name__)
+
+# Callback invoked when a single payload fails to decode: (payload_type, exc).
+OnError = Callable[[PayloadType, Exception], None]
+
+PAYLOAD_DECODERS: Dict[PayloadType, Callable[[str], DecodedPayload]] = {
+    PayloadType.PROMPT_TOKEN_IDS: decode_prompt_token_ids,
+    PayloadType.LOGPROBS: decode_logprobs,
+    PayloadType.ROUTER_REPLAY: decode_router_replay,
+}
+
+
+def decode_payload(payload_type: PayloadType | str, data_b64: str) -> DecodedPayload:
+    """Decode a single payload by type.
+
+    ``payload_type`` accepts a ``PayloadType`` or its string value, so external
+    callers can pass either.
+    """
+    ptype = PayloadType(payload_type)
+    decoder = PAYLOAD_DECODERS.get(ptype)
+    if decoder is None:
+        raise ValueError(f"No decoder registered for payload type: {ptype!r}")
+    return decoder(data_b64)
+
+
+def decode_payloads(
+    payloads: Dict[str, Any],
+    *,
+    on_error: Optional[OnError] = None,
+) -> Dict[PayloadType, DecodedPayload]:
+    """Master decode: run every registered decoder over a gateway ``payloads`` dict.
+
+    Args:
+        payloads: The ``payloads`` object from a gateway trace (i.e.
+            ``trace["payloads"]``), mapping payload-type name -> ``{"manifest", "data"}``.
+        on_error: Optional callback ``(payload_type, exc)`` invoked when a present
+            payload fails to decode. Defaults to logging a warning. A failure in
+            one payload never blocks the others.
+
+    Returns:
+        ``{PayloadType: DecodedPayload}`` for every payload that is present and
+        decodes successfully. Only known ``PayloadType`` members are considered,
+        so unknown payload types from a newer gateway are ignored rather than
+        raising.
+    """
+    if not isinstance(payloads, dict):
+        return {}
+
+    decoded: Dict[PayloadType, DecodedPayload] = {}
+    for ptype, decoder in PAYLOAD_DECODERS.items():
+        entry = payloads.get(ptype)
+        if not isinstance(entry, dict) or not entry.get("data"):
+            continue
+        try:
+            decoded[ptype] = decoder(entry["data"])
+        except Exception as exc:  # noqa: BLE001 - isolate per-payload failures
+            if on_error is not None:
+                on_error(ptype, exc)
+            else:
+                logger.warning("Failed to decode %s payload: %s", ptype.value, exc)
+    return decoded
+
+
+def decode_trace(
+    trace: Dict[str, Any],
+    *,
+    on_error: Optional[OnError] = None,
+) -> Dict[PayloadType, DecodedPayload]:
+    """Convenience wrapper around :func:`decode_payloads` for a raw trace dict."""
+    payloads = trace.get("payloads") if isinstance(trace, dict) else None
+    return decode_payloads(payloads or {}, on_error=on_error)

eval_protocol-0.3.30/eval_protocol/adapters/r3_deserializer.py → eval_protocol-0.3.32/eval_protocol/tracing/router_replay.py

@@ -20,6 +20,8 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import zstandard as zstd
 
+from .types import DecodedPayload, PayloadType
+
 MAGIC = b"R3V1"
 HEADER_FORMAT = "<4sBBBBIIIIQ"
 HEADER_SIZE = struct.calcsize(HEADER_FORMAT)  # 32 bytes
@@ -185,3 +187,16 @@ def decompress_and_parse_r3(
         matrices[pos] = base64.b64encode(matrix_bytes[start:end]).decode("ascii")
 
     return matrices, metadata
+
+
+def decode_router_replay(data_b64: str) -> DecodedPayload:
+    """Decode a gateway ``payloads.router_replay.data`` blob into a ``DecodedPayload``.
+
+    ``value`` is the per-token ``List[Optional[str]]`` of base64 routing matrices.
+    """
+    matrices, metadata = decompress_and_parse_r3(data_b64)
+    return DecodedPayload(
+        payload_type=PayloadType.ROUTER_REPLAY,
+        value=matrices,
+        metadata=metadata,
+    )

eval_protocol-0.3.32/eval_protocol/tracing/types.py

@@ -0,0 +1,41 @@
+"""Shared types for the Fireworks tracing-gateway payload decoders."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class PayloadType(str, Enum):
+    """Known out-of-band trace payload types emitted by the tracing gateway.
+
+    Canonical source of truth for payload-type names on the EP side. Mirrors the
+    gateway's ``rft_tracing.schemas.PayloadType`` but is defined locally so this
+    package has no dependency on the gateway/mono codebase. Being a ``str`` enum,
+    members compare and hash equal to their string value, so they can be used
+    directly against the gateway's string-keyed ``payloads`` JSON.
+    """
+
+    PROMPT_TOKEN_IDS = "prompt_token_ids"
+    LOGPROBS = "logprobs"
+    ROUTER_REPLAY = "router_replay"
+
+
+@dataclass(frozen=True)
+class DecodedPayload:
+    """A single decoded gateway payload.
+
+    ``value`` shape depends on ``payload_type``:
+      - ``PROMPT_TOKEN_IDS`` -> ``List[int]``
+      - ``LOGPROBS``         -> ``List[float]`` (per completion token); the
+        optional per-token ids are in ``token_ids``
+      - ``ROUTER_REPLAY``    -> ``List[Optional[str]]`` (per-token base64 routing
+        matrices, ``None`` where absent)
+    """
+
+    payload_type: PayloadType
+    value: Any
+    metadata: Dict[str, Any]
+    # LOGPROBS only: per-completion-token ids, or None if any were missing.
+    token_ids: Optional[List[int]] = None

{eval_protocol-0.3.30 → eval_protocol-0.3.32/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.30
+Version: 0.3.32
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.30 → eval_protocol-0.3.32}/eval_protocol.egg-info/SOURCES.txt

@@ -263,6 +263,12 @@ eval_protocol/rewards/repetition.py
 eval_protocol/rewards/tag_count.py
 eval_protocol/stats/__init__.py
 eval_protocol/stats/confidence_intervals.py
+eval_protocol/tracing/__init__.py
+eval_protocol/tracing/logprobs.py
+eval_protocol/tracing/prompt_token_ids.py
+eval_protocol/tracing/registry.py
+eval_protocol/tracing/router_replay.py
+eval_protocol/tracing/types.py
 eval_protocol/training/__init__.py
 eval_protocol/training/gepa_trainer.py
 eval_protocol/training/gepa_utils.py