eval-protocol 0.2.46.dev2__py3-none-any.whl → 0.2.46.dev3__py3-none-any.whl

eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-10T01:09:04-0700",
+ "date": "2025-10-10T01:45:32-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "859dce2af419fc26a8634c1254a75a52d9c246d4",
- "version": "0.2.46-dev2"
+ "full-revisionid": "1757548441eb93afd5dc0428b0218637787cdd80",
+ "version": "0.2.46-dev3"
 }
 '''  # END VERSION_JSON
 

eval_protocol/adapters/fireworks_tracing.py

@@ -343,11 +343,11 @@ class FireworksTracingAdapter(BaseAdapter):
         # Remove None values
         params = {k: v for k, v in params.items() if v is not None}
 
-        # Make request to proxy
+        # Make request to proxy (using pointwise for efficiency)
         if self.project_id:
-            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces"
+            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces/pointwise"
         else:
-            url = f"{self.base_url}/v1/traces"
+            url = f"{self.base_url}/v1/traces/pointwise"
 
         headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
 
@@ -367,7 +367,7 @@ class FireworksTracingAdapter(BaseAdapter):
                 except Exception:  # In case e.response.json() fails
                     error_msg = f"Proxy error: {e.response.text}"
 
-            logger.error("Failed to fetch traces from proxy: %s", error_msg)
+            logger.error("Failed to fetch traces from proxy (HTTP %s): %s", e.response.status_code, error_msg)
             return eval_rows
         except requests.exceptions.RequestException as e:
             # Non-HTTP errors (network issues, timeouts, etc.)

eval_protocol/proxy/proxy_core/app.py

@@ -16,7 +16,7 @@ from contextlib import asynccontextmanager
 from .models import ProxyConfig, LangfuseTracesResponse, TracesParams, ChatParams, ChatRequestHook, TracesRequestHook
 from .auth import AuthProvider, NoAuthProvider
 from .litellm import handle_chat_completion, proxy_to_litellm
-from .langfuse import fetch_langfuse_traces
+from .langfuse import fetch_langfuse_traces, pointwise_fetch_langfuse_trace
 
 # Configure logging before any other imports (so all modules inherit this config)
 log_level = os.getenv("LOG_LEVEL", "INFO").upper()
@@ -267,6 +267,27 @@ def create_app(
             params=params,
         )
 
+    @app.get("/traces/pointwise", response_model=LangfuseTracesResponse)
+    @app.get("/v1/traces/pointwise", response_model=LangfuseTracesResponse)
+    @app.get("/project_id/{project_id}/traces/pointwise", response_model=LangfuseTracesResponse)
+    @app.get("/v1/project_id/{project_id}/traces/pointwise", response_model=LangfuseTracesResponse)
+    async def pointwise_get_langfuse_trace(
+        request: Request,
+        params: TracesParams = Depends(get_traces_params),
+        project_id: Optional[str] = None,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+        _: None = Depends(require_auth),
+    ) -> LangfuseTracesResponse:
+        if project_id is not None:
+            params.project_id = project_id
+        return await pointwise_fetch_langfuse_trace(
+            config=config,
+            redis_client=redis_client,
+            request=request,
+            params=params,
+        )
+
     # Health
     @app.get("/health")
     async def health():

eval_protocol/proxy/proxy_core/langfuse.py

@@ -366,3 +366,161 @@ async def fetch_langfuse_traces(
         raise
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error fetching traces from Langfuse: {str(e)}")
+
+
+async def pointwise_fetch_langfuse_trace(
+    config: ProxyConfig,
+    redis_client: redis.Redis,
+    request: Request,
+    params: TracesParams,
+):
+    """
+    Fetch the latest trace from Langfuse for the specified project.
+
+    Since insertion_ids are UUID v7 (time-ordered), we only fetch the last one
+    as it contains all accumulated information from the pointwise evaluation.
+
+    Returns a single trace object or raises if not found.
+    """
+
+    # Preprocess traces request
+    if config.preprocess_traces_request:
+        params = config.preprocess_traces_request(request, params)
+
+    tags = params.tags
+    project_id = params.project_id
+    user_id = params.user_id
+    session_id = params.session_id
+    name = params.name
+    environment = params.environment
+    version = params.version
+    release = params.release
+    fields = params.fields
+    hours_back = params.hours_back
+    from_timestamp = params.from_timestamp
+    to_timestamp = params.to_timestamp
+    sleep_between_gets = params.sleep_between_gets
+    max_retries = params.max_retries
+
+    # Use default project if not specified
+    if project_id is None:
+        project_id = config.default_project_id
+
+    # Validate project_id
+    if project_id not in config.langfuse_keys:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Project ID '{project_id}' not found. Available projects: {list(config.langfuse_keys.keys())}",
+        )
+
+    # Extract rollout_id from tags for Redis lookup
+    rollout_id = _extract_tag_value(tags, "rollout_id:")
+
+    try:
+        # Import the Langfuse adapter
+        from langfuse import Langfuse
+
+        # Create Langfuse client with the project's keys
+        logger.debug(f"Connecting to Langfuse at {config.langfuse_host} for project '{project_id}'")
+        langfuse_client = Langfuse(
+            public_key=config.langfuse_keys[project_id]["public_key"],
+            secret_key=config.langfuse_keys[project_id]["secret_key"],
+            host=config.langfuse_host,
+        )
+
+        # Parse datetime strings if provided
+        from_ts = None
+        to_ts = None
+        if from_timestamp:
+            from_ts = datetime.fromisoformat(from_timestamp.replace("Z", "+00:00"))
+        if to_timestamp:
+            to_ts = datetime.fromisoformat(to_timestamp.replace("Z", "+00:00"))
+
+        # Determine time window: explicit from/to takes precedence over hours_back
+        if from_ts is None and to_ts is None and hours_back:
+            to_ts = datetime.now()
+            from_ts = to_ts - timedelta(hours=hours_back)
+
+        # Get expected insertion_ids from Redis for completeness checking
+        expected_ids: Set[str] = set()
+        if rollout_id:
+            expected_ids = get_insertion_ids(redis_client, rollout_id)
+            logger.info(f"Pointwise fetch for rollout_id '{rollout_id}', expecting {len(expected_ids)} insertion_ids")
+            if not expected_ids:
+                logger.warning(
+                    f"No expected insertion_ids found in Redis for rollout '{rollout_id}'. Returning empty trace."
+                )
+                raise HTTPException(
+                    status_code=500,
+                    detail=f"No expected insertion_ids found in Redis for rollout '{rollout_id}'. Returning empty trace.",
+                )
+
+        # Get the latest (last) insertion_id since UUID v7 is time-ordered
+        latest_insertion_id = max(expected_ids)  # UUID v7 max = newest
+        logger.info(f"Targeting latest insertion_id (last5): {latest_insertion_id[-5:]} for rollout '{rollout_id}'")
+
+        for retry in range(max_retries):
+            # Fetch trace list targeting the latest insertion_id
+            traces = await _fetch_trace_list_with_retry(
+                langfuse_client,
+                page=1,
+                limit=1,  # Only need the one trace
+                tags=[f"insertion_id:{latest_insertion_id}"],
+                user_id=user_id,
+                session_id=session_id,
+                name=name,
+                environment=environment,
+                version=version,
+                release=release,
+                fields=fields,
+                from_ts=from_ts,
+                to_ts=to_ts,
+                max_retries=max_retries,
+            )
+
+            if traces and traces.data:
+                # Get the trace info
+                trace_info = traces.data[0]
+                logger.debug(f"Found trace {trace_info.id} for latest insertion_id {latest_insertion_id[-5:]}")
+
+                # Fetch full trace details
+                trace_full = await _fetch_trace_detail_with_retry(
+                    langfuse_client,
+                    trace_info.id,
+                    max_retries,
+                )
+
+                if trace_full:
+                    trace_dict = _serialize_trace_to_dict(trace_full)
+                    logger.info(
+                        f"Successfully fetched latest trace for rollout '{rollout_id}', insertion_id (last5): {latest_insertion_id[-5:]}"
+                    )
+                    return LangfuseTracesResponse(
+                        project_id=project_id,
+                        total_traces=1,
+                        traces=[TraceResponse(**trace_dict)],
+                    )
+
+            # If not successful and not last retry, sleep and continue
+            if retry < max_retries - 1:
+                wait_time = 2**retry
+                logger.info(
+                    f"Pointwise fetch attempt {retry + 1}/{max_retries} failed for rollout '{rollout_id}', insertion_id (last5): {latest_insertion_id[-5:]}. Retrying in {wait_time}s..."
+                )
+                await asyncio.sleep(wait_time)
+
+        # After all retries failed
+        logger.error(
+            f"Failed to fetch latest trace for rollout '{rollout_id}', insertion_id (last5): {latest_insertion_id[-5:]} after {max_retries} retries"
+        )
+        raise HTTPException(
+            status_code=404,
+            detail=f"Failed to fetch latest trace for rollout '{rollout_id}' after {max_retries} retries",
+        )
+
+    except ImportError:
+        raise HTTPException(status_code=500, detail="Langfuse SDK not installed. Install with: pip install langfuse")
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching latest trace from Langfuse: {str(e)}")

{eval_protocol-0.2.46.dev2.dist-info → eval_protocol-0.2.46.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.46.dev2
+Version: 0.2.46.dev3
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.46.dev2.dist-info → eval_protocol-0.2.46.dev3.dist-info}/RECORD

@@ -5,7 +5,7 @@ development/utils/generate_api_key.py,sha256=hHCMFkzW4yxqwcn2ct5diDm-PR9cMX9XP7I
 development/utils/subprocess_manager.py,sha256=7n7rT9ji7h93i79SMrGS5RNesrnLFjdFON9_eQCmYNE,18937
 eval_protocol/__init__.py,sha256=bPjRmbNv6MggQz8dq_KSmSKwGSwyLGvuPenS_H8lsoc,3987
 eval_protocol/__main__.py,sha256=FIW5fo2X2rsPThurSlZEuqvE1u0XNwJW1uoej_OhAAs,161
-eval_protocol/_version.py,sha256=gzKeHivAEoFj1hOoUFImQJrTSWPqo5-Ca7KHlw5Au80,503
+eval_protocol/_version.py,sha256=JSNgYjGNXILDIicPuCTagxr-Bfge7f0ZiyNv_OqEVbg,503
 eval_protocol/auth.py,sha256=Yvpx-DkxcXYvezmcuNl9BFws2A9o9u8iLbIu4mBUUhE,8463
 eval_protocol/cli.py,sha256=0J7RJKT_4aWYy02yB9Q6gm-RxvlRSt6dxwM1fW-pCpE,22989
 eval_protocol/common_utils.py,sha256=eRzO_SQNfxBz-vIF9PdTVFAzuAb2zfuCnKSKY_xdKQY,2491
@@ -30,7 +30,7 @@ eval_protocol/adapters/__init__.py,sha256=yLGzY9Gcr6KsIgN36fUc_4HRpnEUL2bkBu7pC-
 eval_protocol/adapters/base.py,sha256=7-0nj76caW0E2COWnWFIkRamlOCUeK0xWbd_Bpyt-iU,783
 eval_protocol/adapters/bigquery.py,sha256=vYMc9JTUFX2jyQ70W7TOHBjttGscyJQLdMcEQD36I08,11432
 eval_protocol/adapters/braintrust.py,sha256=b2BRpecpqszECMPUm5pXjX0BW8Bwyhis39EuIqbH-NY,11422
-eval_protocol/adapters/fireworks_tracing.py,sha256=i6EilIwPPUy6oAsyIPqXmhbSvtXeSxBNMZKwV8ULseg,15399
+eval_protocol/adapters/fireworks_tracing.py,sha256=9DwoclXLBsuI-Rr5iJ5I-MXiGmp3iMIUQvLv2CXI9BM,15486
 eval_protocol/adapters/huggingface.py,sha256=6UJHXKI8wBkf_1eLlkgoucmhQuoVAMRW-NsXiKAwzqs,15228
 eval_protocol/adapters/langchain.py,sha256=LJWz6KMrX2svTbejdMbzHWb913bWCA1ElsG95FHFxLs,8822
 eval_protocol/adapters/langfuse.py,sha256=u6vDXHfGITUEqftTBJJiJz0GcvTWXR8kKtouMbgyfU4,22870
@@ -149,9 +149,9 @@ eval_protocol/mcp_servers/tau2/tests/system_prompts/mock_agent_system_prompt.md,
 eval_protocol/mcp_servers/tau2/tests/system_prompts/retail_agent_system_prompt.md,sha256=GRSWoBs9xJwMjJXezCfyuT70pvEovEG7J4bie3tyS8o,7061
 eval_protocol/proxy/__init__.py,sha256=HkIDs5jsSA1gf-Too0ekzKwXOfuRISASe0dIwiEVWMs,416
 eval_protocol/proxy/proxy_core/__init__.py,sha256=O30kHfOExmPgf4jXuO2l5_nOYhC-jiSVAc6ndlfomH8,265
-eval_protocol/proxy/proxy_core/app.py,sha256=5sWByGaru6MDktfzrnyqIOYAoc1CxgRqf6-dj4HFwvI,10898
+eval_protocol/proxy/proxy_core/app.py,sha256=bDruTeGbF9_oVcbyyDOn1FBRwEbML7et_mQGE8aLlCY,11892
 eval_protocol/proxy/proxy_core/auth.py,sha256=ymuPE0pXegiWCm5L9LrCTmdMU3YQ_eD5YYXEwSgYUjk,463
-eval_protocol/proxy/proxy_core/langfuse.py,sha256=4TX84FBgs2v4FHmjxPkUD5K1J2M334BCTqkxJkAmcwI,15011
+eval_protocol/proxy/proxy_core/langfuse.py,sha256=3o38KjwNy4pQ6Cat4AomkTs_kJDzZcM9pnI6Gaa25xE,21396
 eval_protocol/proxy/proxy_core/litellm.py,sha256=LhYUvK9PywmBrBJLDMYeNjqTE0f-ST35AkdxXVyDg9Y,5949
 eval_protocol/proxy/proxy_core/main.py,sha256=8f5jxhT-dGLRVdEVrWkvJjHirpV6O4oQiZxHVYAuHHU,266
 eval_protocol/proxy/proxy_core/models.py,sha256=R0LU_daiZlcuTCYAt5q521fDRsNjMWeeLy0iYpZg-Go,2707
@@ -230,7 +230,7 @@ eval_protocol/utils/show_results_url.py,sha256=PHM6dWtCUiuV5WQgvHegnxY7ofkE4b9wO
 eval_protocol/utils/static_policy.py,sha256=fiKnOS06EG5OB6p5An_yY_dLAvVboYnC4Sqx5z_v3-g,10716
 eval_protocol/utils/subprocess_utils.py,sha256=2EcoVNLSlfdxwQn-2pscqjiGpBR4Ho8kfRnmzmew-1w,3504
 eval_protocol/utils/vite_server.py,sha256=0Tfh1LfTqYpFZxkO2syrF5I0cBEJHFmYXd2N4CWkca8,5051
-eval_protocol-0.2.46.dev2.dist-info/licenses/LICENSE,sha256=OzeIb507xW9AVhGMqqHpoL_EFRJUo8Sb7A3LN5NqFfQ,1075
+eval_protocol-0.2.46.dev3.dist-info/licenses/LICENSE,sha256=OzeIb507xW9AVhGMqqHpoL_EFRJUo8Sb7A3LN5NqFfQ,1075
 vendor/tau2/__init__.py,sha256=EQMX_v8x-YBV24ia35_nLkf5MrC6aAuT_M5m7IJcl3k,541
 vendor/tau2/cli.py,sha256=lhJocXCDxEfdv7gIxya5b0w5J5qebpgrg_ZTpjGp_ww,7515
 vendor/tau2/config.py,sha256=LrkKRGSFH4Cvf9CNO-MttJMvIia0a2zP1uKVnUQi6B8,1278
@@ -330,8 +330,8 @@ vite-app/dist/assets/index-C81y9r9l.js,sha256=7fWlEMpE0hAZHAQ4gHEnv1yNu_STClSwsY
 vite-app/dist/assets/index-C81y9r9l.js.map,sha256=MkiPbSe9_T6HHqVZOdpIde-LcujdcNWkc4uEEwgfyk8,3862571
 vite-app/dist/assets/index-DpYZaoAr.css,sha256=v5t6cVU5X1gnABnBT6RBLRTBbrrFbJ92rYFqzaNoUkg,24878
 vite-app/dist/assets/logo-light-BprIBJQW.png,sha256=rRXC24eqrQO3y--N493THrD48WQVAhSVMHM_iDKy250,21694
-eval_protocol-0.2.46.dev2.dist-info/METADATA,sha256=r2beRO4vHNtn1TEakiGJHsFMjMnhW8vApBz0MrbGQCI,7498
-eval_protocol-0.2.46.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-eval_protocol-0.2.46.dev2.dist-info/entry_points.txt,sha256=CebRaxbWXly21zPN1fbyAw26kNUU2dv7zZyGkXxtFVw,183
-eval_protocol-0.2.46.dev2.dist-info/top_level.txt,sha256=8jjn7dpvLPL4RX2JBeAfPPMOR6x6f7E4o4yFiKLEHuw,33
-eval_protocol-0.2.46.dev2.dist-info/RECORD,,
+eval_protocol-0.2.46.dev3.dist-info/METADATA,sha256=gqHozwmc1Anw3jeTU1t1EmDYM3Ybh4XdZEWCEuG9_3Y,7498
+eval_protocol-0.2.46.dev3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+eval_protocol-0.2.46.dev3.dist-info/entry_points.txt,sha256=CebRaxbWXly21zPN1fbyAw26kNUU2dv7zZyGkXxtFVw,183
+eval_protocol-0.2.46.dev3.dist-info/top_level.txt,sha256=8jjn7dpvLPL4RX2JBeAfPPMOR6x6f7E4o4yFiKLEHuw,33
+eval_protocol-0.2.46.dev3.dist-info/RECORD,,