eval-protocol 0.2.46.dev0__py3-none-any.whl → 0.2.46.dev2__py3-none-any.whl

eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-10T00:24:47-0700",
+ "date": "2025-10-10T01:09:04-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "98d81a75608e700f91e697f88c149641328d9507",
- "version": "0.2.46-dev"
+ "full-revisionid": "859dce2af419fc26a8634c1254a75a52d9c246d4",
+ "version": "0.2.46-dev2"
 }
 '''  # END VERSION_JSON
 

eval_protocol/proxy/proxy_core/langfuse.py

@@ -163,11 +163,6 @@ async def fetch_langfuse_traces(
     This endpoint uses the stored Langfuse keys for the project and polls
     traces based on the provided filters.
 
-    SECURITY:
-    - Tags are REQUIRED and must not be empty
-    - At least one tag MUST be in the format 'rollout_id:*'
-    - This prevents accidentally fetching all traces or traces from other clients
-
     If project_id is not provided, uses the default project.
 
     Returns a list of full trace objects (including observations) in JSON format.
@@ -236,6 +231,7 @@ async def fetch_langfuse_traces(
         expected_ids: Set[str] = set()
         if rollout_id:
             expected_ids = get_insertion_ids(redis_client, rollout_id)
+            logger.info(f"Fetching traces for rollout_id '{rollout_id}', expecting {len(expected_ids)} insertion_ids")
             if not expected_ids:
                 logger.warning(
                     f"No expected insertion_ids found in Redis for rollout '{rollout_id}'. Returning empty traces."
@@ -258,7 +254,9 @@ async def fetch_langfuse_traces(
                 # Build targeted tags for missing insertion_ids
                 missing_ids = expected_ids - insertion_ids
                 fetch_tags = [f"insertion_id:{id}" for id in missing_ids]
-                logger.info(f"Retry {retry}: Targeting {len(fetch_tags)} missing insertion_ids")
+                logger.info(
+                    f"Retry {retry}: Targeting {len(fetch_tags)} missing insertion_ids for rollout '{rollout_id}' (last5): {[id[-5:] for id in sorted(missing_ids)[:10]]}{'...' if len(missing_ids) > 10 else ''}"
+                )
 
             current_page = 1
             collected = 0
@@ -313,6 +311,7 @@ async def fetch_langfuse_traces(
                             insertion_id = _extract_tag_value(trace_dict.get("tags", []), "insertion_id:")
                             if insertion_id:
                                 insertion_ids.add(insertion_id)
+                                logger.debug(f"Found insertion_id '{insertion_id}' for rollout '{rollout_id}'")
 
                         except Exception as e:
                             logger.warning("Failed to serialize trace %s: %s", trace_info.id, e)
@@ -331,8 +330,12 @@ async def fetch_langfuse_traces(
 
             # If we have all expected completions or more, return traces. At least once is ok.
             if expected_ids <= insertion_ids:
+                logger.info(
+                    f"Traces complete for rollout '{rollout_id}': {len(insertion_ids)}/{len(expected_ids)} insertion_ids found, returning {len(all_traces)} traces"
+                )
                 if sample_size is not None and len(all_traces) > sample_size:
                     all_traces = random.sample(all_traces, sample_size)
+                    logger.info(f"Sampled down to {sample_size} traces")
 
                 return LangfuseTracesResponse(
                     project_id=project_id,
@@ -343,8 +346,9 @@ async def fetch_langfuse_traces(
             # If it doesn't match, wait and do loop again (exponential backoff)
             if retry < max_retries - 1:
                 wait_time = 2**retry
+                still_missing = expected_ids - insertion_ids
                 logger.info(
-                    f"Attempt {retry + 1}/{max_retries}. Found {len(insertion_ids)}/{len(expected_ids)} expected. Waiting {wait_time}s..."
+                    f"Attempt {retry + 1}/{max_retries}. Found {len(insertion_ids)}/{len(expected_ids)} for rollout '{rollout_id}'. Still missing (last5): {[id[-5:] for id in sorted(still_missing)[:10]]}{'...' if len(still_missing) > 10 else ''}. Waiting {wait_time}s..."
                 )
                 await asyncio.sleep(wait_time)
 

{eval_protocol-0.2.46.dev0.dist-info → eval_protocol-0.2.46.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.46.dev0
+Version: 0.2.46.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.46.dev0.dist-info → eval_protocol-0.2.46.dev2.dist-info}/RECORD

@@ -5,7 +5,7 @@ development/utils/generate_api_key.py,sha256=hHCMFkzW4yxqwcn2ct5diDm-PR9cMX9XP7I
 development/utils/subprocess_manager.py,sha256=7n7rT9ji7h93i79SMrGS5RNesrnLFjdFON9_eQCmYNE,18937
 eval_protocol/__init__.py,sha256=bPjRmbNv6MggQz8dq_KSmSKwGSwyLGvuPenS_H8lsoc,3987
 eval_protocol/__main__.py,sha256=FIW5fo2X2rsPThurSlZEuqvE1u0XNwJW1uoej_OhAAs,161
-eval_protocol/_version.py,sha256=_SDM03Pv7DVWTRnnye3u_vquFScY59uFLQiEJQ_zMRs,502
+eval_protocol/_version.py,sha256=gzKeHivAEoFj1hOoUFImQJrTSWPqo5-Ca7KHlw5Au80,503
 eval_protocol/auth.py,sha256=Yvpx-DkxcXYvezmcuNl9BFws2A9o9u8iLbIu4mBUUhE,8463
 eval_protocol/cli.py,sha256=0J7RJKT_4aWYy02yB9Q6gm-RxvlRSt6dxwM1fW-pCpE,22989
 eval_protocol/common_utils.py,sha256=eRzO_SQNfxBz-vIF9PdTVFAzuAb2zfuCnKSKY_xdKQY,2491
@@ -151,7 +151,7 @@ eval_protocol/proxy/__init__.py,sha256=HkIDs5jsSA1gf-Too0ekzKwXOfuRISASe0dIwiEVW
 eval_protocol/proxy/proxy_core/__init__.py,sha256=O30kHfOExmPgf4jXuO2l5_nOYhC-jiSVAc6ndlfomH8,265
 eval_protocol/proxy/proxy_core/app.py,sha256=5sWByGaru6MDktfzrnyqIOYAoc1CxgRqf6-dj4HFwvI,10898
 eval_protocol/proxy/proxy_core/auth.py,sha256=ymuPE0pXegiWCm5L9LrCTmdMU3YQ_eD5YYXEwSgYUjk,463
-eval_protocol/proxy/proxy_core/langfuse.py,sha256=jyGBvSR1hspiYD63ebvvNJwCS8tjCYSDsrCAj1BclCQ,14344
+eval_protocol/proxy/proxy_core/langfuse.py,sha256=4TX84FBgs2v4FHmjxPkUD5K1J2M334BCTqkxJkAmcwI,15011
 eval_protocol/proxy/proxy_core/litellm.py,sha256=LhYUvK9PywmBrBJLDMYeNjqTE0f-ST35AkdxXVyDg9Y,5949
 eval_protocol/proxy/proxy_core/main.py,sha256=8f5jxhT-dGLRVdEVrWkvJjHirpV6O4oQiZxHVYAuHHU,266
 eval_protocol/proxy/proxy_core/models.py,sha256=R0LU_daiZlcuTCYAt5q521fDRsNjMWeeLy0iYpZg-Go,2707
@@ -230,7 +230,7 @@ eval_protocol/utils/show_results_url.py,sha256=PHM6dWtCUiuV5WQgvHegnxY7ofkE4b9wO
 eval_protocol/utils/static_policy.py,sha256=fiKnOS06EG5OB6p5An_yY_dLAvVboYnC4Sqx5z_v3-g,10716
 eval_protocol/utils/subprocess_utils.py,sha256=2EcoVNLSlfdxwQn-2pscqjiGpBR4Ho8kfRnmzmew-1w,3504
 eval_protocol/utils/vite_server.py,sha256=0Tfh1LfTqYpFZxkO2syrF5I0cBEJHFmYXd2N4CWkca8,5051
-eval_protocol-0.2.46.dev0.dist-info/licenses/LICENSE,sha256=OzeIb507xW9AVhGMqqHpoL_EFRJUo8Sb7A3LN5NqFfQ,1075
+eval_protocol-0.2.46.dev2.dist-info/licenses/LICENSE,sha256=OzeIb507xW9AVhGMqqHpoL_EFRJUo8Sb7A3LN5NqFfQ,1075
 vendor/tau2/__init__.py,sha256=EQMX_v8x-YBV24ia35_nLkf5MrC6aAuT_M5m7IJcl3k,541
 vendor/tau2/cli.py,sha256=lhJocXCDxEfdv7gIxya5b0w5J5qebpgrg_ZTpjGp_ww,7515
 vendor/tau2/config.py,sha256=LrkKRGSFH4Cvf9CNO-MttJMvIia0a2zP1uKVnUQi6B8,1278
@@ -330,8 +330,8 @@ vite-app/dist/assets/index-C81y9r9l.js,sha256=7fWlEMpE0hAZHAQ4gHEnv1yNu_STClSwsY
 vite-app/dist/assets/index-C81y9r9l.js.map,sha256=MkiPbSe9_T6HHqVZOdpIde-LcujdcNWkc4uEEwgfyk8,3862571
 vite-app/dist/assets/index-DpYZaoAr.css,sha256=v5t6cVU5X1gnABnBT6RBLRTBbrrFbJ92rYFqzaNoUkg,24878
 vite-app/dist/assets/logo-light-BprIBJQW.png,sha256=rRXC24eqrQO3y--N493THrD48WQVAhSVMHM_iDKy250,21694
-eval_protocol-0.2.46.dev0.dist-info/METADATA,sha256=UBJD5ufPLp3Hig1O70i-glLNPb9zM41jHvGFSGKOiME,7498
-eval_protocol-0.2.46.dev0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-eval_protocol-0.2.46.dev0.dist-info/entry_points.txt,sha256=CebRaxbWXly21zPN1fbyAw26kNUU2dv7zZyGkXxtFVw,183
-eval_protocol-0.2.46.dev0.dist-info/top_level.txt,sha256=8jjn7dpvLPL4RX2JBeAfPPMOR6x6f7E4o4yFiKLEHuw,33
-eval_protocol-0.2.46.dev0.dist-info/RECORD,,
+eval_protocol-0.2.46.dev2.dist-info/METADATA,sha256=r2beRO4vHNtn1TEakiGJHsFMjMnhW8vApBz0MrbGQCI,7498
+eval_protocol-0.2.46.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+eval_protocol-0.2.46.dev2.dist-info/entry_points.txt,sha256=CebRaxbWXly21zPN1fbyAw26kNUU2dv7zZyGkXxtFVw,183
+eval_protocol-0.2.46.dev2.dist-info/top_level.txt,sha256=8jjn7dpvLPL4RX2JBeAfPPMOR6x6f7E4o4yFiKLEHuw,33
+eval_protocol-0.2.46.dev2.dist-info/RECORD,,