judgeval 0.16.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-07T20:43:52+00:00
+#   timestamp: 2025-10-09T00:16:42+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -94,6 +94,7 @@ class ResolveProjectNameRequest(TypedDict):
 
 class ResolveProjectNameResponse(TypedDict):
     project_id: str
+    project_created: bool
 
 
 class TraceIdRequest(TypedDict):

judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-07T20:43:51+00:00
+#   timestamp: 2025-10-09T00:16:41+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -101,6 +101,7 @@ class ResolveProjectNameRequest(BaseModel):
 
 class ResolveProjectNameResponse(BaseModel):
     project_id: Annotated[str, Field(title="Project Id")]
+    project_created: Annotated[bool, Field(title="Project Created")]
 
 
 class TraceIdRequest(BaseModel):

judgeval/logger.py

@@ -2,7 +2,7 @@ import logging
 import sys
 
 from judgeval.env import JUDGMENT_NO_COLOR
-from judgeval.utils.decorators import use_once
+from judgeval.utils.decorators.use_once import use_once
 
 RESET = "\033[0m"
 RED = "\033[31m"

judgeval/tracer/__init__.py

@@ -55,7 +55,7 @@ from judgeval.tracer.managers import (
     sync_agent_context,
     async_agent_context,
 )
-from judgeval.utils.decorators import dont_throw
+from judgeval.utils.decorators.dont_throw import dont_throw
 from judgeval.utils.guards import expect_api_key, expect_organization_id
 from judgeval.utils.serialize import safe_serialize
 from judgeval.utils.meta import SingletonMeta
@@ -159,11 +159,14 @@ class Tracer(metaclass=SingletonMeta):
 
         self.judgment_processor = NoOpJudgmentSpanProcessor()
         if self.enable_monitoring:
-            project_id = Tracer._resolve_project_id(
+            project_id, project_created = Tracer._resolve_project_id(
                 self.project_name, self.api_key, self.organization_id
-            )
-
+            ) or (None, False)
             if project_id:
+                if project_created:
+                    judgeval_logger.info(
+                        f"Project {self.project_name} was autocreated successfully."
+                    )
                 self.judgment_processor = self.get_processor(
                     tracer=self,
                     project_name=self.project_name,
@@ -179,7 +182,7 @@ class Tracer(metaclass=SingletonMeta):
                 set_tracer_provider(provider)
             else:
                 judgeval_logger.error(
-                    f"Failed to resolve project {self.project_name}, please create it first at https://app.judgmentlabs.ai/org/{self.organization_id}/projects. Skipping Judgment export."
+                    f"Failed to resolve or autocreate project {self.project_name}, please create it first at https://app.judgmentlabs.ai/org/{self.organization_id}/projects. Skipping Judgment export."
                 )
 
         self.tracer = get_tracer_provider().get_tracer(
@@ -237,14 +240,14 @@ class Tracer(metaclass=SingletonMeta):
     @staticmethod
     def _resolve_project_id(
         project_name: str, api_key: str, organization_id: str
-    ) -> str | None:
+    ) -> Tuple[str, bool]:
         """Resolve project_id from project_name using the API."""
         client = JudgmentSyncClient(
             api_key=api_key,
             organization_id=organization_id,
         )
         response = client.projects_resolve({"project_name": project_name})
-        return response["project_id"]
+        return response["project_id"], response["project_created"]
 
     def get_current_span(self):
         return get_current_span()

judgeval/tracer/keys.py

@@ -12,6 +12,8 @@ class AttributeKeys(str, Enum):
     JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"
     JUDGMENT_UPDATE_ID = "judgment.update_id"
 
+    JUDGMENT_USAGE_METADATA = "judgment.usage.metadata"
+
     JUDGMENT_CUSTOMER_ID = "judgment.customer_id"
 
     JUDGMENT_AGENT_ID = "judgment.agent_id"
@@ -31,13 +33,15 @@ class AttributeKeys(str, Enum):
     GEN_AI_SYSTEM = "gen_ai.system"
     GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
     GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
-    GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
+    GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = (
+        "gen_ai.usage.cache_creation_input_tokens"
+    )
+    GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"
+
     GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
     GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
     GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"
 
-    GEN_AI_USAGE_TOTAL_COST = "gen_ai.usage.total_cost_usd"
-
 
 class InternalAttributeKeys(str, Enum):
     """