judgeval 0.16.0__tar.gz → 0.16.1__tar.gz

{judgeval-0.16.0 → judgeval-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.16.0
+Version: 0.16.1
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.16.0 → judgeval-0.16.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.16.0"
+version = "0.16.1"
 authors = [
     { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
     { name = "Alex Shan", email = "alex@judgmentlabs.ai" },

{judgeval-0.16.0 → judgeval-0.16.1}/src/judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-07T20:43:52+00:00
+#   timestamp: 2025-10-09T00:16:42+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -94,6 +94,7 @@ class ResolveProjectNameRequest(TypedDict):
 
 class ResolveProjectNameResponse(TypedDict):
     project_id: str
+    project_created: bool
 
 
 class TraceIdRequest(TypedDict):

{judgeval-0.16.0 → judgeval-0.16.1}/src/judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-07T20:43:51+00:00
+#   timestamp: 2025-10-09T00:16:41+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -101,6 +101,7 @@ class ResolveProjectNameRequest(BaseModel):
 
 class ResolveProjectNameResponse(BaseModel):
     project_id: Annotated[str, Field(title="Project Id")]
+    project_created: Annotated[bool, Field(title="Project Created")]
 
 
 class TraceIdRequest(BaseModel):

{judgeval-0.16.0 → judgeval-0.16.1}/src/judgeval/logger.py

@@ -2,7 +2,7 @@ import logging
 import sys
 
 from judgeval.env import JUDGMENT_NO_COLOR
-from judgeval.utils.decorators import use_once
+from judgeval.utils.decorators.use_once import use_once
 
 RESET = "\033[0m"
 RED = "\033[31m"

{judgeval-0.16.0 → judgeval-0.16.1}/src/judgeval/tracer/__init__.py

@@ -55,7 +55,7 @@ from judgeval.tracer.managers import (
     sync_agent_context,
     async_agent_context,
 )
-from judgeval.utils.decorators import dont_throw
+from judgeval.utils.decorators.dont_throw import dont_throw
 from judgeval.utils.guards import expect_api_key, expect_organization_id
 from judgeval.utils.serialize import safe_serialize
 from judgeval.utils.meta import SingletonMeta
@@ -159,11 +159,14 @@ class Tracer(metaclass=SingletonMeta):
 
         self.judgment_processor = NoOpJudgmentSpanProcessor()
         if self.enable_monitoring:
-            project_id = Tracer._resolve_project_id(
+            project_id, project_created = Tracer._resolve_project_id(
                 self.project_name, self.api_key, self.organization_id
-            )
-
+            ) or (None, False)
             if project_id:
+                if project_created:
+                    judgeval_logger.info(
+                        f"Project {self.project_name} was autocreated successfully."
+                    )
                 self.judgment_processor = self.get_processor(
                     tracer=self,
                     project_name=self.project_name,
@@ -179,7 +182,7 @@ class Tracer(metaclass=SingletonMeta):
                 set_tracer_provider(provider)
             else:
                 judgeval_logger.error(
-                    f"Failed to resolve project {self.project_name}, please create it first at https://app.judgmentlabs.ai/org/{self.organization_id}/projects. Skipping Judgment export."
+                    f"Failed to resolve or autocreate project {self.project_name}, please create it first at https://app.judgmentlabs.ai/org/{self.organization_id}/projects. Skipping Judgment export."
                 )
 
         self.tracer = get_tracer_provider().get_tracer(
@@ -237,14 +240,14 @@ class Tracer(metaclass=SingletonMeta):
     @staticmethod
     def _resolve_project_id(
         project_name: str, api_key: str, organization_id: str
-    ) -> str | None:
+    ) -> Tuple[str, bool]:
         """Resolve project_id from project_name using the API."""
         client = JudgmentSyncClient(
             api_key=api_key,
             organization_id=organization_id,
         )
         response = client.projects_resolve({"project_name": project_name})
-        return response["project_id"]
+        return response["project_id"], response["project_created"]
 
     def get_current_span(self):
         return get_current_span()

{judgeval-0.16.0 → judgeval-0.16.1}/src/judgeval/tracer/keys.py

@@ -12,6 +12,8 @@ class AttributeKeys(str, Enum):
     JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"
     JUDGMENT_UPDATE_ID = "judgment.update_id"
 
+    JUDGMENT_USAGE_METADATA = "judgment.usage.metadata"
+
     JUDGMENT_CUSTOMER_ID = "judgment.customer_id"
 
     JUDGMENT_AGENT_ID = "judgment.agent_id"
@@ -31,13 +33,15 @@ class AttributeKeys(str, Enum):
     GEN_AI_SYSTEM = "gen_ai.system"
     GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
     GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
-    GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
+    GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = (
+        "gen_ai.usage.cache_creation_input_tokens"
+    )
+    GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"
+
     GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
     GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
     GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"
 
-    GEN_AI_USAGE_TOTAL_COST = "gen_ai.usage.total_cost_usd"
-
 
 class InternalAttributeKeys(str, Enum):
     """

judgeval-0.16.1/src/judgeval/tracer/llm/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+from .config import _detect_provider, wrap_provider
+
+
+__all__ = ["_detect_provider", "wrap_provider"]

judgeval-0.16.1/src/judgeval/tracer/llm/config.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+from judgeval.logger import judgeval_logger
+
+from judgeval.tracer.llm.constants import ProviderType
+from judgeval.tracer.llm.providers import (
+    HAS_OPENAI,
+    HAS_TOGETHER,
+    HAS_ANTHROPIC,
+    HAS_GOOGLE_GENAI,
+    HAS_GROQ,
+    ApiClient,
+)
+
+if TYPE_CHECKING:
+    from judgeval.tracer import Tracer
+
+
+def _detect_provider(client: ApiClient) -> ProviderType:
+    if HAS_OPENAI:
+        from judgeval.tracer.llm.providers import openai_OpenAI, openai_AsyncOpenAI
+
+        assert openai_OpenAI is not None, "OpenAI client not found"
+        assert openai_AsyncOpenAI is not None, "OpenAI async client not found"
+        if isinstance(client, (openai_OpenAI, openai_AsyncOpenAI)):
+            return ProviderType.OPENAI
+
+    if HAS_ANTHROPIC:
+        from judgeval.tracer.llm.providers import (
+            anthropic_Anthropic,
+            anthropic_AsyncAnthropic,
+        )
+
+        assert anthropic_Anthropic is not None, "Anthropic client not found"
+        assert anthropic_AsyncAnthropic is not None, "Anthropic async client not found"
+        if isinstance(client, (anthropic_Anthropic, anthropic_AsyncAnthropic)):
+            return ProviderType.ANTHROPIC
+
+    if HAS_TOGETHER:
+        from judgeval.tracer.llm.providers import (
+            together_Together,
+            together_AsyncTogether,
+        )
+
+        assert together_Together is not None, "Together client not found"
+        assert together_AsyncTogether is not None, "Together async client not found"
+        if isinstance(client, (together_Together, together_AsyncTogether)):
+            return ProviderType.TOGETHER
+
+    if HAS_GOOGLE_GENAI:
+        from judgeval.tracer.llm.providers import (
+            google_genai_Client,
+            google_genai_AsyncClient,
+        )
+
+        assert google_genai_Client is not None, "Google GenAI client not found"
+        assert google_genai_AsyncClient is not None, (
+            "Google GenAI async client not found"
+        )
+        if isinstance(client, (google_genai_Client, google_genai_AsyncClient)):
+            return ProviderType.GOOGLE
+
+    if HAS_GROQ:
+        from judgeval.tracer.llm.providers import groq_Groq, groq_AsyncGroq
+
+        assert groq_Groq is not None, "Groq client not found"
+        assert groq_AsyncGroq is not None, "Groq async client not found"
+        if isinstance(client, (groq_Groq, groq_AsyncGroq)):
+            return ProviderType.GROQ
+
+    judgeval_logger.warning(
+        f"Unknown client type {type(client)}, Trying to wrap as OpenAI-compatible. "
+        "If this is a mistake or you think we should support this client, please file an issue at https://github.com/JudgmentLabs/judgeval/issues!"
+    )
+
+    return ProviderType.DEFAULT
+
+
+def wrap_provider(tracer: Tracer, client: ApiClient) -> ApiClient:
+    """
+    Wraps an API client to add tracing capabilities.
+    Supports OpenAI, Together, Anthropic, Google GenAI, and Groq clients.
+    """
+    provider_type = _detect_provider(client)
+
+    if provider_type == ProviderType.OPENAI:
+        from .llm_openai.wrapper import wrap_openai_client
+
+        return wrap_openai_client(tracer, client)
+    elif provider_type == ProviderType.ANTHROPIC:
+        from .llm_anthropic.wrapper import wrap_anthropic_client
+
+        return wrap_anthropic_client(tracer, client)
+    elif provider_type == ProviderType.TOGETHER:
+        from .llm_together.wrapper import wrap_together_client
+
+        return wrap_together_client(tracer, client)
+    elif provider_type == ProviderType.GOOGLE:
+        from .llm_google.wrapper import wrap_google_client
+
+        return wrap_google_client(tracer, client)
+    elif provider_type == ProviderType.GROQ:
+        from .llm_groq.wrapper import wrap_groq_client
+
+        return wrap_groq_client(tracer, client)
+    else:
+        # Default to OpenAI-compatible wrapping for unknown clients
+        from .llm_openai.wrapper import wrap_openai_client
+
+        return wrap_openai_client(tracer, client)

judgeval-0.16.1/src/judgeval/tracer/llm/constants.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class ProviderType(Enum):
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+    TOGETHER = "together"
+    GOOGLE = "google"
+    GROQ = "groq"
+    DEFAULT = "default"

judgeval-0.16.1/src/judgeval/tracer/llm/llm_anthropic/__init__.py

@@ -0,0 +1,3 @@
+from .wrapper import wrap_anthropic_client
+
+__all__ = ["wrap_anthropic_client"]