openlit 1.34.29__py3-none-any.whl → 1.34.31__py3-none-any.whl

openlit/__init__.py

@@ -137,7 +137,9 @@ def module_exists(module_name):
     return True
 
 
-def instrument_if_available(instrumentor_name, instrumentor_instance, config, disabled_instrumentors):
+def instrument_if_available(
+    instrumentor_name, instrumentor_instance, config, disabled_instrumentors
+):
     """Instruments the specified instrumentor if its library is available."""
     if instrumentor_name in disabled_instrumentors:
         logger.info("Instrumentor %s is disabled", instrumentor_name)
@@ -242,13 +244,13 @@ def init(
 
         # Setup events based on the provided or default configuration.
         event_provider = setup_events(
-                application_name=application_name,
-                environment=environment,
-                event_logger=event_logger,
-                otlp_endpoint=None,
-                otlp_headers=None,
-                disable_batch=disable_batch,
-            )
+            application_name=application_name,
+            environment=environment,
+            event_logger=event_logger,
+            otlp_endpoint=None,
+            otlp_headers=None,
+            disable_batch=disable_batch,
+        )
 
         if not event_provider:
             logger.error("OpenLIT events setup failed. Events will not be available")
@@ -268,8 +270,11 @@ def init(
             )
             return
 
-        if os.getenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "").lower == "false":
-            capture_message_content=False
+        if (
+            os.getenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "").lower
+            == "false"
+        ):
+            capture_message_content = False
 
         # Update global configuration with the provided settings.
         config.update_config(
@@ -462,9 +467,7 @@ def trace(wrapped):
                     SERVICE_NAME,
                     OpenlitConfig.application_name,
                 )
-                span.set_attribute(
-                    DEPLOYMENT_ENVIRONMENT, OpenlitConfig.environment
-                )
+                span.set_attribute(DEPLOYMENT_ENVIRONMENT, OpenlitConfig.environment)
             except Exception as meta_exception:
                 logging.error(
                     "Failed to set metadata for %s: %s",

openlit/_instrumentors.py

@@ -1,6 +1,7 @@
 """
 OpenLIT Instrumentors
 """
+
 import importlib
 
 # Mapping of instrumentor names to their required Python packages
@@ -49,7 +50,7 @@ MODULE_NAME_MAP = {
     "letta": "letta",
     "together": "together",
     "openai-agents": "agents",
-    "pydantic_ai": "pydantic_ai"
+    "pydantic_ai": "pydantic_ai",
 }
 
 # Dictionary mapping instrumentor names to their full module paths

openlit/evals/all.py

@@ -11,11 +11,14 @@ from openlit.evals.utils import (
     llm_response,
     parse_llm_response,
     eval_metrics,
-    eval_metric_attributes
+    eval_metric_attributes,
 )
 
-def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
-                      threshold_score: Optional[float] = 0.5) -> str:
+
+def get_system_prompt(
+    custom_categories: Optional[Dict[str, str]] = None,
+    threshold_score: Optional[float] = 0.5,
+) -> str:
     """
     Returns the system prompt used for LLM analysis, including custom categories if provided.
 
@@ -51,8 +54,12 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
 
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {value}" for key, value in custom_categories.items()])
-        base_prompt += f"\n    Additional Hallucination Categories:\n{custom_categories_str}"
+        custom_categories_str = "\n".join(
+            [f"- {key}: {value}" for key, value in custom_categories.items()]
+        )
+        base_prompt += (
+            f"\n    Additional Hallucination Categories:\n{custom_categories_str}"
+        )
 
     base_prompt += """
 
@@ -96,6 +103,7 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
     return base_prompt
 
+
 class All:
     """
     A class to detect Bias, Hallucination and Toxicity in AI responses using LLM or custom categories.
@@ -108,11 +116,16 @@ class All:
         custom_categories (Optional[Dict[str, str]]): Additional categories for prompt injections.
     """
 
-    def __init__(self, provider: Optional[str] = "openai", api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 collect_metrics: Optional[bool] = False,
-                 threshold_score: Optional[float] = 0.5):
+    def __init__(
+        self,
+        provider: Optional[str] = "openai",
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        collect_metrics: Optional[bool] = False,
+        threshold_score: Optional[float] = 0.5,
+    ):
         """
         Initializes the All Evals detector with specified LLM settings, custom rules, and categories.
 
@@ -130,15 +143,22 @@ class All:
         self.provider = provider
         if self.provider is None:
             raise ValueError("An LLM provider must be specified evaluation.")
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.collect_metrics = collect_metrics
         self.custom_categories = custom_categories
         self.threshold_score = threshold_score
-        self.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)
-
-    def measure(self, prompt: Optional[str] = "",
-               contexts: Optional[List[str]] = None,
-               text: Optional[str] = None) -> JsonOutput:
+        self.system_prompt = get_system_prompt(
+            self.custom_categories, self.threshold_score
+        )
+
+    def measure(
+        self,
+        prompt: Optional[str] = "",
+        contexts: Optional[List[str]] = None,
+        text: Optional[str] = None,
+    ) -> JsonOutput:
         """
         Detects toxicity in AI output using LLM or custom rules.
 
@@ -156,14 +176,23 @@ class All:
         llm_result = parse_llm_response(response)
         result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
 
-        result = JsonOutput(score=llm_result.score, evaluation=llm_result.evaluation,
-                            classification=llm_result.classification,
-                            explanation=llm_result.explanation, verdict=result_verdict)
+        result = JsonOutput(
+            score=llm_result.score,
+            evaluation=llm_result.evaluation,
+            classification=llm_result.classification,
+            explanation=llm_result.explanation,
+            verdict=result_verdict,
+        )
 
         if self.collect_metrics:
             eval_counter = eval_metrics()
-            attributes = eval_metric_attributes(result_verdict, result.score, result.evaluation,
-                                                result.classification, result.explanation)
+            attributes = eval_metric_attributes(
+                result_verdict,
+                result.score,
+                result.evaluation,
+                result.classification,
+                result.explanation,
+            )
             eval_counter.add(1, attributes)
 
         return result

openlit/evals/bias_detection.py

@@ -11,11 +11,14 @@ from openlit.evals.utils import (
     llm_response,
     parse_llm_response,
     eval_metrics,
-    eval_metric_attributes
+    eval_metric_attributes,
 )
 
-def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
-                      threshold_score: Optional[float] = 0.5) -> str:
+
+def get_system_prompt(
+    custom_categories: Optional[Dict[str, str]] = None,
+    threshold_score: Optional[float] = 0.5,
+) -> str:
     """
     Returns the system prompt used for LLM analysis, including custom categories if provided.
 
@@ -55,7 +58,9 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
 
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {value}" for key, value in custom_categories.items()])
+        custom_categories_str = "\n".join(
+            [f"- {key}: {value}" for key, value in custom_categories.items()]
+        )
         base_prompt += f"\n    Additional Bias Categories:\n{custom_categories_str}"
 
     base_prompt += """
@@ -100,6 +105,7 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
     return base_prompt
 
+
 class BiasDetector:
     """
     A class to detect Bias in AI responses using LLM or custom categories.
@@ -112,11 +118,16 @@ class BiasDetector:
         custom_categories (Optional[Dict[str, str]]): Additional categories for prompt injections.
     """
 
-    def __init__(self, provider: Optional[str] = "openai", api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 collect_metrics: Optional[bool] = False,
-                 threshold_score: Optional[float] = 0.5):
+    def __init__(
+        self,
+        provider: Optional[str] = "openai",
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        collect_metrics: Optional[bool] = False,
+        threshold_score: Optional[float] = 0.5,
+    ):
         """
         Initializes the Bias detector with specified LLM settings, custom rules, and categories.
 
@@ -134,15 +145,22 @@ class BiasDetector:
         self.provider = provider
         if self.provider is None:
             raise ValueError("An LLM provider must be specified for Bias detection.")
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.collect_metrics = collect_metrics
         self.custom_categories = custom_categories
         self.threshold_score = threshold_score
-        self.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)
-
-    def measure(self, prompt: Optional[str] = "",
-               contexts: Optional[List[str]] = None,
-               text: Optional[str] = None) -> JsonOutput:
+        self.system_prompt = get_system_prompt(
+            self.custom_categories, self.threshold_score
+        )
+
+    def measure(
+        self,
+        prompt: Optional[str] = "",
+        contexts: Optional[List[str]] = None,
+        text: Optional[str] = None,
+    ) -> JsonOutput:
         """
         Detects toxicity in AI output using LLM or custom rules.
 
@@ -160,14 +178,23 @@ class BiasDetector:
         llm_result = parse_llm_response(response)
         result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
 
-        result = JsonOutput(score=llm_result.score, evaluation=llm_result.evaluation,
-                            classification=llm_result.classification,
-                            explanation=llm_result.explanation, verdict=result_verdict)
+        result = JsonOutput(
+            score=llm_result.score,
+            evaluation=llm_result.evaluation,
+            classification=llm_result.classification,
+            explanation=llm_result.explanation,
+            verdict=result_verdict,
+        )
 
         if self.collect_metrics:
             eval_counter = eval_metrics()
-            attributes = eval_metric_attributes(result_verdict, result.score, result.evaluation,
-                                                result.classification, result.explanation)
+            attributes = eval_metric_attributes(
+                result_verdict,
+                result.score,
+                result.evaluation,
+                result.classification,
+                result.explanation,
+            )
             eval_counter.add(1, attributes)
 
         return result

openlit/evals/hallucination.py

@@ -11,12 +11,15 @@ from openlit.evals.utils import (
     llm_response,
     parse_llm_response,
     eval_metrics,
-    eval_metric_attributes
+    eval_metric_attributes,
 )
 
+
 # pylint: disable=unused-argument
-def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
-                      threshold_score: Optional[float] = 0.5) -> str:
+def get_system_prompt(
+    custom_categories: Optional[Dict[str, str]] = None,
+    threshold_score: Optional[float] = 0.5,
+) -> str:
     """
     Returns the system prompt used for LLM analysis, including custom categories if provided.
 
@@ -49,8 +52,12 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
 
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {value}" for key, value in custom_categories.items()])
-        base_prompt += f"\n    Additional Hallucination Categories:\n{custom_categories_str}"
+        custom_categories_str = "\n".join(
+            [f"- {key}: {value}" for key, value in custom_categories.items()]
+        )
+        base_prompt += (
+            f"\n    Additional Hallucination Categories:\n{custom_categories_str}"
+        )
 
     base_prompt += """
 
@@ -98,6 +105,7 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
 
     return base_prompt
 
+
 class Hallucination:
     """
     A class to detect hallucinations in AI responses using LLM or custom categories.
@@ -110,11 +118,16 @@ class Hallucination:
         custom_categories (Optional[Dict[str, str]]): Additional categories for prompt injections.
     """
 
-    def __init__(self, provider: Optional[str] = "openai", api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 collect_metrics: Optional[bool] = False,
-                 threshold_score: Optional[float] = 0.5):
+    def __init__(
+        self,
+        provider: Optional[str] = "openai",
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        collect_metrics: Optional[bool] = False,
+        threshold_score: Optional[float] = 0.5,
+    ):
         """
         Initializes the Hallucination detector with specified LLM settings, custom rules, and categories.
 
@@ -131,16 +144,25 @@ class Hallucination:
 
         self.provider = provider
         if self.provider is None:
-            raise ValueError("An LLM provider must be specified for Hallucination detection.")
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+            raise ValueError(
+                "An LLM provider must be specified for Hallucination detection."
+            )
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.collect_metrics = collect_metrics
         self.custom_categories = custom_categories
         self.threshold_score = threshold_score
-        self.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)
-
-    def measure(self, prompt: Optional[str] = "",
-               contexts: Optional[List[str]] = None,
-               text: Optional[str] = None) -> JsonOutput:
+        self.system_prompt = get_system_prompt(
+            self.custom_categories, self.threshold_score
+        )
+
+    def measure(
+        self,
+        prompt: Optional[str] = "",
+        contexts: Optional[List[str]] = None,
+        text: Optional[str] = None,
+    ) -> JsonOutput:
         """
         Detects hallucinations in AI output using LLM or custom rules.
 
@@ -157,14 +179,23 @@ class Hallucination:
         response = llm_response(self.provider, llm_prompt, self.model, self.base_url)
         llm_result = parse_llm_response(response)
         result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
-        result = JsonOutput(score=llm_result.score, evaluation=llm_result.evaluation,
-                            classification=llm_result.classification,
-                            explanation=llm_result.explanation, verdict=result_verdict)
+        result = JsonOutput(
+            score=llm_result.score,
+            evaluation=llm_result.evaluation,
+            classification=llm_result.classification,
+            explanation=llm_result.explanation,
+            verdict=result_verdict,
+        )
 
         if self.collect_metrics:
             eval_counter = eval_metrics()
-            attributes = eval_metric_attributes(result_verdict, result.score, result.evaluation,
-                                                result.classification, result.explanation)
+            attributes = eval_metric_attributes(
+                result_verdict,
+                result.score,
+                result.evaluation,
+                result.classification,
+                result.explanation,
+            )
             eval_counter.add(1, attributes)
 
         return result

openlit/evals/toxicity.py

@@ -11,11 +11,14 @@ from openlit.evals.utils import (
     llm_response,
     parse_llm_response,
     eval_metrics,
-    eval_metric_attributes
+    eval_metric_attributes,
 )
 
-def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
-                      threshold_score: Optional[float] = 0.5) -> str:
+
+def get_system_prompt(
+    custom_categories: Optional[Dict[str, str]] = None,
+    threshold_score: Optional[float] = 0.5,
+) -> str:
     """
     Returns the system prompt used for LLM analysis, including custom categories if provided.
 
@@ -50,7 +53,9 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
 
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {value}" for key, value in custom_categories.items()])
+        custom_categories_str = "\n".join(
+            [f"- {key}: {value}" for key, value in custom_categories.items()]
+        )
         base_prompt += f"\n    Additional Toxicity Categories:\n{custom_categories_str}"
 
     base_prompt += """
@@ -95,6 +100,7 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,
     """
     return base_prompt
 
+
 class ToxicityDetector:
     """
     A class to detect Toxicity in AI responses using LLM or custom categories.
@@ -107,11 +113,16 @@ class ToxicityDetector:
         custom_categories (Optional[Dict[str, str]]): Additional categories for prompt injections.
     """
 
-    def __init__(self, provider: Optional[str] = "openai", api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 collect_metrics: Optional[bool] = False,
-                 threshold_score: Optional[float] = 0.5):
+    def __init__(
+        self,
+        provider: Optional[str] = "openai",
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        collect_metrics: Optional[bool] = False,
+        threshold_score: Optional[float] = 0.5,
+    ):
         """
         Initializes the toxicity detector with specified LLM settings, custom rules, and categories.
 
@@ -128,16 +139,25 @@ class ToxicityDetector:
 
         self.provider = provider
         if self.provider is None:
-            raise ValueError("An LLM provider must be specified for toxicity detection.")
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+            raise ValueError(
+                "An LLM provider must be specified for toxicity detection."
+            )
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.collect_metrics = collect_metrics
         self.custom_categories = custom_categories
         self.threshold_score = threshold_score
-        self.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)
-
-    def measure(self, prompt: Optional[str] = "",
-               contexts: Optional[List[str]] = "",
-               text: Optional[str] = None) -> JsonOutput:
+        self.system_prompt = get_system_prompt(
+            self.custom_categories, self.threshold_score
+        )
+
+    def measure(
+        self,
+        prompt: Optional[str] = "",
+        contexts: Optional[List[str]] = "",
+        text: Optional[str] = None,
+    ) -> JsonOutput:
         """
         Detects toxicity in AI output using LLM or custom rules.
 
@@ -155,14 +175,23 @@ class ToxicityDetector:
         llm_result = parse_llm_response(response)
         result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
 
-        result = JsonOutput(score=llm_result.score, evaluation=llm_result.evaluation,
-                            classification=llm_result.classification,
-                            explanation=llm_result.explanation, verdict=result_verdict)
+        result = JsonOutput(
+            score=llm_result.score,
+            evaluation=llm_result.evaluation,
+            classification=llm_result.classification,
+            explanation=llm_result.explanation,
+            verdict=result_verdict,
+        )
 
         if self.collect_metrics:
             eval_counter = eval_metrics()
-            attributes = eval_metric_attributes(result_verdict, result.score, result.evaluation,
-                                                result.classification, result.explanation)
+            attributes = eval_metric_attributes(
+                result_verdict,
+                result.score,
+                result.evaluation,
+                result.classification,
+                result.explanation,
+            )
             eval_counter.add(1, attributes)
 
         return result