judgeval 0.9.3__py3-none-any.whl → 0.9.4__py3-none-any.whl

judgeval/tracer/__init__.py

@@ -57,7 +57,7 @@ from judgeval.utils.serialize import safe_serialize
 from judgeval.version import get_version
 from judgeval.warnings import JudgmentWarning
 
-from judgeval.tracer.keys import AttributeKeys, ResourceKeys, InternalAttributeKeys
+from judgeval.tracer.keys import AttributeKeys, InternalAttributeKeys
 from judgeval.api import JudgmentSyncClient
 from judgeval.tracer.llm import wrap_provider
 from judgeval.utils.url import url_for
@@ -65,6 +65,7 @@ from judgeval.tracer.local_eval_queue import LocalEvaluationQueue
 from judgeval.tracer.processors import (
     JudgmentSpanProcessor,
     NoOpJudgmentSpanProcessor,
+    NoOpSpanProcessor,
 )
 from judgeval.tracer.utils import set_span_attribute, TraceScorerConfig
 
@@ -85,19 +86,6 @@ class AgentContext(TypedDict):
     parent_agent_id: str | None
 
 
-def resolve_project_id(
-    api_key: str, organization_id: str, project_name: str
-) -> str | None:
-    try:
-        client = JudgmentSyncClient(
-            api_key=api_key,
-            organization_id=organization_id,
-        )
-        return client.projects_resolve({"project_name": project_name})["project_id"]
-    except Exception:
-        return None
-
-
 class Tracer:
     _active_tracers: List[Tracer] = []
 
@@ -188,38 +176,20 @@ class Tracer:
         self.cost_context = ContextVar("current_cost_context", default=None)
 
         if self.enable_monitoring:
-            project_id = resolve_project_id(
-                self.api_key, self.organization_id, self.project_name
-            )
-
-            resource_attributes = resource_attributes or {}
-            resource_attributes.update(
-                {
-                    ResourceKeys.SERVICE_NAME: self.project_name,
-                    ResourceKeys.TELEMETRY_SDK_NAME: "judgeval",
-                    ResourceKeys.TELEMETRY_SDK_VERSION: get_version(),
-                }
-            )
-
-            if project_id is not None:
-                resource_attributes[ResourceKeys.JUDGMENT_PROJECT_ID] = project_id
-            else:
-                judgeval_logger.error(
-                    f"Failed to resolve project {self.project_name}, please create it first at https://app.judgmentlabs.ai/projects. Skipping Judgment export."
-                )
-
-            resource = Resource.create(resource_attributes)
-
             self.judgment_processor = JudgmentSpanProcessor(
                 self,
-                self.api_url,
+                self.project_name,
                 self.api_key,
                 self.organization_id,
                 max_queue_size=2**18,
                 export_timeout_millis=30000,
+                resource_attributes=resource_attributes,
             )
-            self.processors.append(self.judgment_processor)
+
+            resource = Resource.create(self.judgment_processor.resource_attributes)
             self.provider = TracerProvider(resource=resource)
+
+            self.processors.append(self.judgment_processor)
             for processor in self.processors:
                 self.provider.add_span_processor(processor)
 
@@ -253,6 +223,14 @@ class Tracer:
     def get_current_cost_context(self):
         return self.cost_context
 
+    def get_processor(self):
+        """Get the judgment span processor instance.
+
+        Returns:
+            The JudgmentSpanProcessor or NoOpJudgmentSpanProcessor instance used by this tracer.
+        """
+        return self.judgment_processor
+
     def set_customer_id(self, customer_id: str) -> None:
         span = self.get_current_span()
         if span and span.is_recording():
@@ -913,11 +891,7 @@ class Tracer:
         proper cleanup before program termination.
         """
         try:
-            success = self.force_flush(timeout_millis=30000)
-            if not success:
-                judgeval_logger.warning(
-                    "Some spans may not have been exported before program exit"
-                )
+            self.force_flush(timeout_millis=30000)
         except Exception as e:
             judgeval_logger.warning(f"Error during atexit flush: {e}")
 
@@ -1074,3 +1048,13 @@ def format_inputs(
         return inputs
     except Exception:
         return {}
+
+
+# Export processor classes for direct access
+__all__ = [
+    "Tracer",
+    "wrap",
+    "JudgmentSpanProcessor",
+    "NoOpJudgmentSpanProcessor",
+    "NoOpSpanProcessor",
+]

judgeval/tracer/processors/__init__.py

@@ -6,8 +6,13 @@ from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor, SpanConte
 from opentelemetry.sdk.trace.export import (
     BatchSpanProcessor,
 )
+from opentelemetry.sdk.resources import Resource
 from judgeval.tracer.exporters import JudgmentSpanExporter
-from judgeval.tracer.keys import AttributeKeys, InternalAttributeKeys
+from judgeval.tracer.keys import AttributeKeys, InternalAttributeKeys, ResourceKeys
+from judgeval.api import JudgmentSyncClient
+from judgeval.logger import judgeval_logger
+from judgeval.utils.url import url_for
+from judgeval.version import get_version
 
 if TYPE_CHECKING:
     from judgeval.tracer import Tracer
@@ -31,15 +36,27 @@ class JudgmentSpanProcessor(BatchSpanProcessor):
     def __init__(
         self,
         tracer: Tracer,
-        endpoint: str,
+        project_name: str,
         api_key: str,
         organization_id: str,
         /,
         *,
         max_queue_size: int = 2**18,
         export_timeout_millis: int = 30000,
+        resource_attributes: Optional[dict[str, Any]] = None,
     ):
         self.tracer = tracer
+        self.project_name = project_name
+        self.api_key = api_key
+        self.organization_id = organization_id
+
+        # Resolve project_id
+        self.project_id = self._resolve_project_id()
+
+        # Set up resource attributes with project_id
+        self._setup_resource_attributes(resource_attributes or {})
+
+        endpoint = url_for("/otel/v1/traces")
         super().__init__(
             JudgmentSpanExporter(
                 endpoint=endpoint,
@@ -53,6 +70,38 @@ class JudgmentSpanProcessor(BatchSpanProcessor):
             defaultdict(dict)
         )
 
+    def _resolve_project_id(self) -> str | None:
+        """Resolve project_id from project_name using the API."""
+        try:
+            client = JudgmentSyncClient(
+                api_key=self.api_key,
+                organization_id=self.organization_id,
+            )
+            return client.projects_resolve({"project_name": self.project_name})[
+                "project_id"
+            ]
+        except Exception:
+            return None
+
+    def _setup_resource_attributes(self, resource_attributes: dict[str, Any]) -> None:
+        """Set up resource attributes including project_id."""
+        resource_attributes.update(
+            {
+                ResourceKeys.SERVICE_NAME: self.project_name,
+                ResourceKeys.TELEMETRY_SDK_NAME: "judgeval",
+                ResourceKeys.TELEMETRY_SDK_VERSION: get_version(),
+            }
+        )
+
+        if self.project_id is not None:
+            resource_attributes[ResourceKeys.JUDGMENT_PROJECT_ID] = self.project_id
+        else:
+            judgeval_logger.error(
+                f"Failed to resolve project {self.project_name}, please create it first at https://app.judgmentlabs.ai/projects. Skipping Judgment export."
+            )
+
+        self.resource_attributes = resource_attributes
+
     def _get_span_key(self, span_context: SpanContext) -> tuple[int, int]:
         return (span_context.trace_id, span_context.span_id)
 
@@ -103,11 +152,18 @@ class JudgmentSpanProcessor(BatchSpanProcessor):
 
         attributes = dict(current_span.attributes or {})
         attributes[AttributeKeys.JUDGMENT_UPDATE_ID] = current_update_id
+
+        existing_resource_attrs = (
+            dict(current_span.resource.attributes) if current_span.resource else {}
+        )
+        merged_resource_attrs = {**existing_resource_attrs, **self.resource_attributes}
+        merged_resource = Resource.create(merged_resource_attrs)
+
         partial_span = ReadableSpan(
             name=current_span.name,
             context=span_context,
             parent=current_span.parent,
-            resource=current_span.resource,
+            resource=merged_resource,
             attributes=attributes,
             events=current_span.events,
             links=current_span.links,
@@ -137,11 +193,20 @@ class JudgmentSpanProcessor(BatchSpanProcessor):
             attributes = dict(span.attributes or {})
             attributes[AttributeKeys.JUDGMENT_UPDATE_ID] = 20
 
+            existing_resource_attrs = (
+                dict(span.resource.attributes) if span.resource else {}
+            )
+            merged_resource_attrs = {
+                **existing_resource_attrs,
+                **self.resource_attributes,
+            }
+            merged_resource = Resource.create(merged_resource_attrs)
+
             final_span = ReadableSpan(
                 name=span.name,
                 context=span.context,
                 parent=span.parent,
-                resource=span.resource,
+                resource=merged_resource,
                 attributes=attributes,
                 events=span.events,
                 links=span.links,
@@ -160,7 +225,7 @@ class JudgmentSpanProcessor(BatchSpanProcessor):
 
 class NoOpJudgmentSpanProcessor(JudgmentSpanProcessor):
     def __init__(self):
-        super().__init__(None, "", "", "")  # type: ignore[arg-type]
+        pass
 
     def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None:
         pass
@@ -177,5 +242,18 @@ class NoOpJudgmentSpanProcessor(JudgmentSpanProcessor):
     def emit_partial(self) -> None:
         pass
 
+    def set_internal_attribute(
+        self, span_context: SpanContext, key: str, value: Any
+    ) -> None:
+        pass
+
+    def get_internal_attribute(
+        self, span_context: SpanContext, key: str, default: Any = None
+    ) -> Any:
+        return default
+
+    def increment_update_id(self, span_context: SpanContext) -> int:
+        return 0
+
 
-__all__ = ("NoOpSpanProcessor", "JudgmentSpanProcessor", "NoOpJudgmentSpanProcessor")
+__all__ = ["NoOpSpanProcessor", "JudgmentSpanProcessor", "NoOpJudgmentSpanProcessor"]

judgeval/utils/serialize.py

@@ -19,6 +19,8 @@ from pydantic import BaseModel
 from pydantic.types import SecretBytes, SecretStr
 import orjson
 
+from judgeval.logger import judgeval_logger
+
 
 """
 This module contains the encoders used by jsonable_encoder to convert Python objects to JSON serializable data types.
@@ -244,4 +246,8 @@ encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE)
 
 # Seralize arbitrary object to a json string
 def safe_serialize(obj: Any) -> str:
-    return orjson.dumps(json_encoder(obj)).decode()
+    try:
+        return orjson.dumps(json_encoder(obj)).decode()
+    except Exception as e:
+        judgeval_logger.warning(f"Error serializing object: {e}")
+        return orjson.dumps(repr(obj)).decode()

{judgeval-0.9.3.dist-info → judgeval-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.9.3
+Version: 0.9.4
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -9,7 +9,7 @@ License-Expression: Apache-2.0
 License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Requires-Dist: boto3>=1.40.11
 Requires-Dist: click<8.2.0
 Requires-Dist: dotenv

{judgeval-0.9.3.dist-info → judgeval-0.9.4.dist-info}/RECORD

@@ -46,7 +46,7 @@ judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=aQ
 judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=C-9Q7s9K7mcgFMcEL0I_7XQZMRqrL5MFRi9G6Dx8-v8,8505
 judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py,sha256=BhrLnIASZOTT9XJ6giYSoVfdR7NYsjRRTOTNioNtEiU,610
 judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py,sha256=bMu0WMJaXdMyDTN42sVLoWV-lrUHCEa8iDrCI_K7nlQ,808
-judgeval/tracer/__init__.py,sha256=WmRmfvBOkzwEaAljwJ_ClNDJt1zB6J562G5BYXNMySY,36914
+judgeval/tracer/__init__.py,sha256=0DM6ixBI75FuVG7UMG_k-KHJm1MyFbRyhAUPm2GYu9A,36057
 judgeval/tracer/constants.py,sha256=ae8tivAW97awJQxdRB9OMqX50wOLX3zqChT_AGkPBu0,85
 judgeval/tracer/keys.py,sha256=qXPoZSkEhVF-YYfQ9-zeDMVdr4GtpPf2W7MPJaN2AQo,2889
 judgeval/tracer/local_eval_queue.py,sha256=Amt7xkdmVJH1l2itm-ogiIW5oDaLnACisGfsdZjazn0,7228
@@ -58,7 +58,7 @@ judgeval/tracer/exporters/store.py,sha256=KQV3cyqteesByQjR-9VdPXT9OlUZ-6F08ogqj8
 judgeval/tracer/exporters/utils.py,sha256=JRcoSQuEHxMDJbXfyrUIfA2SHBVkZM82h4bTbYGxkNw,1154
 judgeval/tracer/llm/__init__.py,sha256=p9uwWPg9k-NcWjj9TbwQj55sHhBOqRYx2-Ld6YHaFUs,42625
 judgeval/tracer/llm/providers.py,sha256=QQLJlSNnDjXRAc2Wqw78o254COJUSXX39D7D_mx3NVA,2651
-judgeval/tracer/processors/__init__.py,sha256=Fpm_cocYj3fwWAE9hcVder6TcYcycbXJWpmdyOjw5uY,5802
+judgeval/tracer/processors/__init__.py,sha256=fjk3zGxQGp6adnj1-QdSaiRJk-VhyzuKG5vCalvbucI,8645
 judgeval/trainer/__init__.py,sha256=h_DDVV7HFF7HUPAJFpt2d9wjqgnmEVcHxqZyB1k7pPQ,257
 judgeval/trainer/config.py,sha256=8s0X8B334PJomorwONaUpb6K8cAMxRdYAeQdtx7HPHs,4258
 judgeval/trainer/console.py,sha256=PJ0rCnDwC7aoW-VsLDS96ZyMyagh-l9EOJKff1ATIpo,4342
@@ -69,12 +69,12 @@ judgeval/utils/decorators.py,sha256=rdqY1w0zNL6O6GU6Wdeo0-x5EgpFTEhU2vkgiWsRYdc,
 judgeval/utils/file_utils.py,sha256=3LI1YCZwO5ogTgJreyOgRgDksey3natO2Td1PQqaPyY,3252
 judgeval/utils/guards.py,sha256=QBb6m6KElxdvt2bskLZCKh_zGHbBcqV-VfGzT63o3hY,807
 judgeval/utils/meta.py,sha256=wQFCLJTNKF9yUdXcw37AT6mC-wqzZpAvjn5gP_6flD8,349
-judgeval/utils/serialize.py,sha256=AtpdMQXC03xhImLJjSNCi-PqsUek2mcwvtd2ryrATTQ,6057
+judgeval/utils/serialize.py,sha256=QXR-8Nj5rqOrI9zLx0oRLdk6DW6Bc7j8eyF4zQ7PLxA,6256
 judgeval/utils/testing.py,sha256=kJOq4LlEXaNThfg9oSIRqSK7IH8AwLgbukjn5uxMY7A,3661
 judgeval/utils/url.py,sha256=Shf0v3XcbaWpL0m1eGJEEO_z4TsQCnDB2Rl25OTUmiI,195
 judgeval/utils/version_check.py,sha256=kcF6SvB6GbVKI0Gv9QRVm-kvBn9_z-c3jmPORsXO3h0,1015
-judgeval-0.9.3.dist-info/METADATA,sha256=vMjBFO5pCziw56JqqnRvqAMJD7WXFdVL3yfIHqADC3c,8869
-judgeval-0.9.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-judgeval-0.9.3.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
-judgeval-0.9.3.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
-judgeval-0.9.3.dist-info/RECORD,,
+judgeval-0.9.4.dist-info/METADATA,sha256=Hr4y27-wt-658_DJd_D7oAUpDebQS5a9jfdQwGvfmbg,8869
+judgeval-0.9.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+judgeval-0.9.4.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
+judgeval-0.9.4.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
+judgeval-0.9.4.dist-info/RECORD,,