judgeval 0.16.7__py3-none-any.whl → 0.16.9__py3-none-any.whl

judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-09T00:16:42+00:00
+#   timestamp: 2025-10-15T19:25:00+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -94,7 +94,6 @@ class ResolveProjectNameRequest(TypedDict):
 
 class ResolveProjectNameResponse(TypedDict):
     project_id: str
-    project_created: bool
 
 
 class TraceIdRequest(TypedDict):

judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-09T00:16:41+00:00
+#   timestamp: 2025-10-15T19:24:59+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -101,7 +101,6 @@ class ResolveProjectNameRequest(BaseModel):
 
 class ResolveProjectNameResponse(BaseModel):
     project_id: Annotated[str, Field(title="Project Id")]
-    project_created: Annotated[bool, Field(title="Project Created")]
 
 
 class TraceIdRequest(BaseModel):

judgeval/tracer/__init__.py

@@ -66,7 +66,6 @@ from judgeval.tracer.keys import AttributeKeys, InternalAttributeKeys
 from judgeval.api import JudgmentSyncClient
 from judgeval.tracer.llm import wrap_provider
 from judgeval.utils.url import url_for
-from judgeval.tracer.local_eval_queue import LocalEvaluationQueue
 from judgeval.tracer.processors import (
     JudgmentSpanProcessor,
     NoOpJudgmentSpanProcessor,
@@ -99,7 +98,6 @@ class Tracer(metaclass=SingletonMeta):
         "enable_evaluation",
         "resource_attributes",
         "api_client",
-        "local_eval_queue",
         "judgment_processor",
         "tracer",
         "agent_context",
@@ -113,7 +111,6 @@ class Tracer(metaclass=SingletonMeta):
     enable_evaluation: bool
     resource_attributes: Optional[Dict[str, Any]]
     api_client: JudgmentSyncClient
-    local_eval_queue: LocalEvaluationQueue
     judgment_processor: JudgmentSpanProcessor
     tracer: ABCTracer
     agent_context: ContextVar[Optional[AgentContext]]
@@ -148,7 +145,6 @@ class Tracer(metaclass=SingletonMeta):
                 api_key=self.api_key,
                 organization_id=self.organization_id,
             )
-            self.local_eval_queue = LocalEvaluationQueue()
 
             if initialize:
                 self.initialize()
@@ -159,14 +155,10 @@ class Tracer(metaclass=SingletonMeta):
 
         self.judgment_processor = NoOpJudgmentSpanProcessor()
         if self.enable_monitoring:
-            project_id, project_created = Tracer._resolve_project_id(
+            project_id = Tracer._resolve_project_id(
                 self.project_name, self.api_key, self.organization_id
-            ) or (None, False)
+            )
             if project_id:
-                if project_created:
-                    judgeval_logger.info(
-                        f"Project {self.project_name} was autocreated successfully."
-                    )
                 self.judgment_processor = self.get_processor(
                     tracer=self,
                     project_name=self.project_name,
@@ -190,9 +182,6 @@ class Tracer(metaclass=SingletonMeta):
             get_version(),
         )
 
-        if self.enable_evaluation and self.enable_monitoring:
-            self.local_eval_queue.start_workers()
-
         self._initialized = True
         atexit.register(self._atexit_flush)
         return self
@@ -240,14 +229,14 @@ class Tracer(metaclass=SingletonMeta):
     @staticmethod
     def _resolve_project_id(
         project_name: str, api_key: str, organization_id: str
-    ) -> Tuple[str, bool]:
+    ) -> str:
         """Resolve project_id from project_name using the API."""
         client = JudgmentSyncClient(
             api_key=api_key,
             organization_id=organization_id,
         )
         response = client.projects_resolve({"project_name": project_name})
-        return response["project_id"], response["project_created"]
+        return response["project_id"]
 
     def get_current_span(self):
         return get_current_span()
@@ -299,6 +288,7 @@ class Tracer(metaclass=SingletonMeta):
         )
         current_agent_context["is_agent_entry_point"] = False
 
+    @dont_throw
     def record_instance_state(self, record_point: Literal["before", "after"], span):
         current_agent_context = self.agent_context.get()
 
@@ -955,45 +945,10 @@ class Tracer(metaclass=SingletonMeta):
                 eval_run.model_dump(warnings=False)  # type: ignore
             )
         else:
-            # Enqueue the evaluation run to the local evaluation queue
-            self.local_eval_queue.enqueue(eval_run)
-
-    def wait_for_completion(self, timeout: Optional[float] = 30.0) -> bool:
-        """Wait for all evaluations and span processing to complete.
-
-        This method blocks until all queued evaluations are processed and
-        all pending spans are flushed to the server.
-
-        Args:
-            timeout: Maximum time to wait in seconds. Defaults to 30 seconds.
-                    None means wait indefinitely.
-
-        Returns:
-            True if all processing completed within the timeout, False otherwise.
-
-        """
-        try:
-            judgeval_logger.debug(
-                "Waiting for all evaluations and spans to complete..."
+            judgeval_logger.warning(
+                "The scorer provided is not hosted, skipping evaluation."
             )
 
-            # Wait for all queued evaluation work to complete
-            eval_completed = self.local_eval_queue.wait_for_completion()
-            if not eval_completed:
-                judgeval_logger.warning(
-                    f"Local evaluation queue did not complete within {timeout} seconds"
-                )
-                return False
-
-            self.force_flush()
-
-            judgeval_logger.debug("All evaluations and spans completed successfully")
-            return True
-
-        except Exception as e:
-            judgeval_logger.warning(f"Error while waiting for completion: {e}")
-            return False
-
 
 def wrap(client: ApiClient) -> ApiClient:
     try:

judgeval/tracer/llm/config.py

@@ -8,7 +8,6 @@ from judgeval.tracer.llm.providers import (
     HAS_TOGETHER,
     HAS_ANTHROPIC,
     HAS_GOOGLE_GENAI,
-    HAS_GROQ,
     ApiClient,
 )
 
@@ -18,55 +17,28 @@ if TYPE_CHECKING:
 
 def _detect_provider(client: ApiClient) -> ProviderType:
     if HAS_OPENAI:
-        from judgeval.tracer.llm.providers import openai_OpenAI, openai_AsyncOpenAI
+        from openai import OpenAI, AsyncOpenAI
 
-        assert openai_OpenAI is not None, "OpenAI client not found"
-        assert openai_AsyncOpenAI is not None, "OpenAI async client not found"
-        if isinstance(client, (openai_OpenAI, openai_AsyncOpenAI)):
+        if isinstance(client, (OpenAI, AsyncOpenAI)):
             return ProviderType.OPENAI
 
     if HAS_ANTHROPIC:
-        from judgeval.tracer.llm.providers import (
-            anthropic_Anthropic,
-            anthropic_AsyncAnthropic,
-        )
-
-        assert anthropic_Anthropic is not None, "Anthropic client not found"
-        assert anthropic_AsyncAnthropic is not None, "Anthropic async client not found"
-        if isinstance(client, (anthropic_Anthropic, anthropic_AsyncAnthropic)):
+        from anthropic import Anthropic, AsyncAnthropic
+
+        if isinstance(client, (Anthropic, AsyncAnthropic)):
             return ProviderType.ANTHROPIC
 
     if HAS_TOGETHER:
-        from judgeval.tracer.llm.providers import (
-            together_Together,
-            together_AsyncTogether,
-        )
-
-        assert together_Together is not None, "Together client not found"
-        assert together_AsyncTogether is not None, "Together async client not found"
-        if isinstance(client, (together_Together, together_AsyncTogether)):
+        from together import Together, AsyncTogether  # type: ignore[import-untyped]
+
+        if isinstance(client, (Together, AsyncTogether)):
             return ProviderType.TOGETHER
 
     if HAS_GOOGLE_GENAI:
-        from judgeval.tracer.llm.providers import (
-            google_genai_Client,
-            google_genai_AsyncClient,
-        )
-
-        assert google_genai_Client is not None, "Google GenAI client not found"
-        assert google_genai_AsyncClient is not None, (
-            "Google GenAI async client not found"
-        )
-        if isinstance(client, (google_genai_Client, google_genai_AsyncClient)):
-            return ProviderType.GOOGLE
+        from google.genai import Client as GoogleClient
 
-    if HAS_GROQ:
-        from judgeval.tracer.llm.providers import groq_Groq, groq_AsyncGroq
-
-        assert groq_Groq is not None, "Groq client not found"
-        assert groq_AsyncGroq is not None, "Groq async client not found"
-        if isinstance(client, (groq_Groq, groq_AsyncGroq)):
-            return ProviderType.GROQ
+        if isinstance(client, GoogleClient):
+            return ProviderType.GOOGLE
 
     judgeval_logger.warning(
         f"Unknown client type {type(client)}, Trying to wrap as OpenAI-compatible. "
@@ -79,7 +51,7 @@ def _detect_provider(client: ApiClient) -> ProviderType:
 def wrap_provider(tracer: Tracer, client: ApiClient) -> ApiClient:
     """
     Wraps an API client to add tracing capabilities.
-    Supports OpenAI, Together, Anthropic, Google GenAI, and Groq clients.
+    Supports OpenAI, Together, Anthropic, and Google GenAI clients.
     """
     provider_type = _detect_provider(client)
 
@@ -99,10 +71,6 @@ def wrap_provider(tracer: Tracer, client: ApiClient) -> ApiClient:
         from .llm_google.wrapper import wrap_google_client
 
         return wrap_google_client(tracer, client)
-    elif provider_type == ProviderType.GROQ:
-        from .llm_groq.wrapper import wrap_groq_client
-
-        return wrap_groq_client(tracer, client)
     else:
         # Default to OpenAI-compatible wrapping for unknown clients
         from .llm_openai.wrapper import wrap_openai_client

judgeval/tracer/llm/constants.py

@@ -6,5 +6,4 @@ class ProviderType(Enum):
     ANTHROPIC = "anthropic"
     TOGETHER = "together"
     GOOGLE = "google"
-    GROQ = "groq"
     DEFAULT = "default"

judgeval/tracer/llm/llm_anthropic/config.py

@@ -1,20 +1,6 @@
 from __future__ import annotations
+import importlib.util
 
-HAS_ANTHROPIC = False
-anthropic_Anthropic = None
-anthropic_AsyncAnthropic = None
+HAS_ANTHROPIC = importlib.util.find_spec("anthropic") is not None
 
-try:
-    from anthropic import Anthropic, AsyncAnthropic  # type: ignore[import-untyped]
-
-    anthropic_Anthropic = Anthropic
-    anthropic_AsyncAnthropic = AsyncAnthropic
-    HAS_ANTHROPIC = True
-except ImportError:
-    pass
-
-__all__ = [
-    "HAS_ANTHROPIC",
-    "anthropic_Anthropic",
-    "anthropic_AsyncAnthropic",
-]
+__all__ = ["HAS_ANTHROPIC"]