openlit 1.34.29__py3-none-any.whl → 1.34.31__py3-none-any.whl

openlit/evals/utils.py

@@ -15,6 +15,7 @@ from openlit.semcov import SemanticConvention
 # Initialize logger for logging potential issues and operations
 logger = logging.getLogger(__name__)
 
+
 class JsonOutput(BaseModel):
     """
     A model representing the structure of JSON output for prompt injection detection.
@@ -32,9 +33,13 @@ class JsonOutput(BaseModel):
     classification: str
     explanation: str
 
-def setup_provider(provider: Optional[str], api_key: Optional[str],
-                   model: Optional[str],
-                   base_url: Optional[str]) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+
+def setup_provider(
+    provider: Optional[str],
+    api_key: Optional[str],
+    model: Optional[str],
+    base_url: Optional[str],
+) -> Tuple[Optional[str], Optional[str], Optional[str]]:
     """
     Sets up the provider, API key, model, and base URL.
 
@@ -52,7 +57,7 @@ def setup_provider(provider: Optional[str], api_key: Optional[str],
     """
     provider_configs = {
         "openai": {"env_var": "OPENAI_API_KEY"},
-        "anthropic": {"env_var": "ANTHROPIC_API_KEY"}
+        "anthropic": {"env_var": "ANTHROPIC_API_KEY"},
     }
 
     if provider is None:
@@ -72,12 +77,16 @@ def setup_provider(provider: Optional[str], api_key: Optional[str],
 
     if not api_key:
         # pylint: disable=line-too-long
-        raise ValueError(f"API key required via 'api_key' parameter or '{env_var}' environment variable")
+        raise ValueError(
+            f"API key required via 'api_key' parameter or '{env_var}' environment variable"
+        )
 
     return api_key, model, base_url
 
 
-def format_prompt(system_prompt: str, prompt: str, contexts: List[str], text: str) -> str:
+def format_prompt(
+    system_prompt: str, prompt: str, contexts: List[str], text: str
+) -> str:
     """
     Format the prompt.
 
@@ -98,6 +107,7 @@ def format_prompt(system_prompt: str, prompt: str, contexts: List[str], text: st
 
     return formatted_prompt
 
+
 def llm_response(provider: str, prompt: str, model: str, base_url: str) -> str:
     """
     Generates an LLM response using the configured provider.
@@ -117,6 +127,7 @@ def llm_response(provider: str, prompt: str, model: str, base_url: str) -> str:
     else:
         raise ValueError(f"Unsupported provider: {provider}")
 
+
 def llm_response_openai(prompt: str, model: str, base_url: str) -> str:
     """
     Interacts with the OpenAI API to get a LLM response.
@@ -142,10 +153,11 @@ def llm_response_openai(prompt: str, model: str, base_url: str) -> str:
             {"role": "user", "content": prompt},
         ],
         temperature=0.0,
-        response_format=JsonOutput
+        response_format=JsonOutput,
     )
     return response.choices[0].message.content
 
+
 def llm_response_anthropic(prompt: str, model: str) -> str:
     """
     Interacts with the Anthropic API to get a LLM response.
@@ -172,23 +184,33 @@ def llm_response_anthropic(prompt: str, model: str) -> str:
                     "verdict": {"type": "string", "description": "Evaluation verdict"},
                     "evaluation": {"type": "string", "description": "Evaluation type"},
                     "score": {"type": "number", "description": "Evaluation score"},
-                    "classification": {"type": "string", "description": "Evaluation category"},
-                    "explanation": {"type": "string", "description": "Evaluation reason"}
+                    "classification": {
+                        "type": "string",
+                        "description": "Evaluation category",
+                    },
+                    "explanation": {
+                        "type": "string",
+                        "description": "Evaluation reason",
+                    },
                 },
-                "required": ["verdict", "evaluation", "score", "classification", "explanation"]
-            }
+                "required": [
+                    "verdict",
+                    "evaluation",
+                    "score",
+                    "classification",
+                    "explanation",
+                ],
+            },
         }
     ]
 
     response = client.messages.create(
         model=model,
-        messages=[
-            {"role": "user", "content": prompt}
-        ],
+        messages=[{"role": "user", "content": prompt}],
         max_tokens=2000,
         temperature=0.0,
         tools=tools,
-        stream=False
+        stream=False,
     )
 
     for content in response.content:
@@ -198,6 +220,7 @@ def llm_response_anthropic(prompt: str, model: str) -> str:
 
     return response
 
+
 def parse_llm_response(response) -> JsonOutput:
     """
     Parses the LLM response into a JsonOutput object.
@@ -220,8 +243,14 @@ def parse_llm_response(response) -> JsonOutput:
         return JsonOutput(**data)
     except (json.JSONDecodeError, TypeError) as e:
         logger.error("Error parsing LLM response: '%s'", e)
-        return JsonOutput(score=0, classification="none", explanation="none",
-                          verdict="no", evaluation="none")
+        return JsonOutput(
+            score=0,
+            classification="none",
+            explanation="none",
+            verdict="no",
+            evaluation="none",
+        )
+
 
 def eval_metrics():
     """
@@ -240,11 +269,12 @@ def eval_metrics():
     guard_requests = meter.create_counter(
         name=SemanticConvention.EVAL_REQUESTS,
         description="Counter for evaluation requests",
-        unit="1"
+        unit="1",
     )
 
     return guard_requests
 
+
 def eval_metric_attributes(verdict, score, validator, classification, explanation):
     """
     Initializes OpenTelemetry attributes for metrics.
@@ -260,16 +290,10 @@ def eval_metric_attributes(verdict, score, validator, classification, explanatio
     """
 
     return {
-            TELEMETRY_SDK_NAME:
-                "openlit",
-            SemanticConvention.EVAL_VERDICT:
-                verdict,
-            SemanticConvention.EVAL_SCORE:
-                score,
-            SemanticConvention.EVAL_VALIDATOR:
-                validator,
-            SemanticConvention.EVAL_CLASSIFICATION:
-                classification,
-            SemanticConvention.EVAL_EXPLANATION:
-                explanation,
+        TELEMETRY_SDK_NAME: "openlit",
+        SemanticConvention.EVAL_VERDICT: verdict,
+        SemanticConvention.EVAL_SCORE: score,
+        SemanticConvention.EVAL_VALIDATOR: validator,
+        SemanticConvention.EVAL_CLASSIFICATION: classification,
+        SemanticConvention.EVAL_EXPLANATION: explanation,
     }

openlit/guard/all.py

@@ -12,10 +12,15 @@ from openlit.guard.utils import (
     parse_llm_response,
     custom_rule_detection,
     guard_metrics,
-    guard_metric_attributes
+    guard_metric_attributes,
 )
 
-def get_all_system_prompt(valid_topics: Optional[List[str]] = None, invalid_topics: Optional[List[str]] = None, custom_categories: Optional[Dict[str, str]] = None) -> str:
+
+def get_all_system_prompt(
+    valid_topics: Optional[List[str]] = None,
+    invalid_topics: Optional[List[str]] = None,
+    custom_categories: Optional[Dict[str, str]] = None,
+) -> str:
     """
     Returns the system prompt used for LLM analysis to capture prompt injections, valid topics, and sensitive topics.
 
@@ -68,8 +73,15 @@ def get_all_system_prompt(valid_topics: Optional[List[str]] = None, invalid_topi
 
     # Append custom categories for prompt injection if provided
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {description}" for key, description in custom_categories.items()])
-        base_prompt += f"\n    Additional Prompt Injection Categories:\n{custom_categories_str}"
+        custom_categories_str = "\n".join(
+            [
+                f"- {key}: {description}"
+                for key, description in custom_categories.items()
+            ]
+        )
+        base_prompt += (
+            f"\n    Additional Prompt Injection Categories:\n{custom_categories_str}"
+        )
 
     base_prompt += """
 
@@ -116,8 +128,15 @@ def get_all_system_prompt(valid_topics: Optional[List[str]] = None, invalid_topi
 
     # Append custom categories for sensitive topics if provided
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {description}" for key, description in custom_categories.items()])
-        base_prompt += f"\n    Additional Sensitive Topics Categories:\n{custom_categories_str}"
+        custom_categories_str = "\n".join(
+            [
+                f"- {key}: {description}"
+                for key, description in custom_categories.items()
+            ]
+        )
+        base_prompt += (
+            f"\n    Additional Sensitive Topics Categories:\n{custom_categories_str}"
+        )
 
     base_prompt += """
 
@@ -126,6 +145,7 @@ def get_all_system_prompt(valid_topics: Optional[List[str]] = None, invalid_topi
     """
     return base_prompt
 
+
 class All:
     """
     A comprehensive class to detect prompt injections, valid/invalid topics, and sensitive topics using LLM or custom rules.
@@ -141,13 +161,18 @@ class All:
         invalid_topics (Optional[List[str]]): List of invalid topics.
     """
 
-    def __init__(self, provider: Optional[str] = None, api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_rules: Optional[List[dict]] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 valid_topics: Optional[List[str]] = None,
-                 invalid_topics: Optional[List[str]] = None,
-                 collect_metrics: Optional[bool] = False):
+    def __init__(
+        self,
+        provider: Optional[str] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_rules: Optional[List[dict]] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        valid_topics: Optional[List[str]] = None,
+        invalid_topics: Optional[List[str]] = None,
+        collect_metrics: Optional[bool] = False,
+    ):
         """
         Initializes the All class with specified LLM settings, custom rules, and categories.
 
@@ -165,8 +190,12 @@ class All:
             ValueError: If provider is not specified.
         """
         self.provider = provider
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
-        self.system_prompt = get_all_system_prompt(valid_topics, invalid_topics, custom_categories)
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
+        self.system_prompt = get_all_system_prompt(
+            valid_topics, invalid_topics, custom_categories
+        )
         self.custom_rules = custom_rules or []
         self.valid_topics = valid_topics or []
         self.invalid_topics = invalid_topics or []
@@ -183,18 +212,31 @@ class All:
             JsonOutput: The structured result of the detection.
         """
         custom_rule_result = custom_rule_detection(text, self.custom_rules)
-        llm_result = JsonOutput(score=0.0, verdict="no", guard="none", classification="none", explanation="none")
+        llm_result = JsonOutput(
+            score=0.0,
+            verdict="no",
+            guard="none",
+            classification="none",
+            explanation="none",
+        )
 
         if self.provider:
             prompt = format_prompt(self.system_prompt, text)
-            llm_result = parse_llm_response(llm_response(self.provider, prompt, self.model, self.base_url))
+            llm_result = parse_llm_response(
+                llm_response(self.provider, prompt, self.model, self.base_url)
+            )
 
         result = max(custom_rule_result, llm_result, key=lambda x: x.score)
 
         if self.collect_metrics:
             guard_counter = guard_metrics()
-            attributes = guard_metric_attributes(result.verdict, result.score, result.guard,
-                                                 result.classification, result.explanation)
+            attributes = guard_metric_attributes(
+                result.verdict,
+                result.score,
+                result.guard,
+                result.classification,
+                result.explanation,
+            )
             guard_counter.add(1, attributes)
 
         return result

openlit/guard/prompt_injection.py

@@ -12,9 +12,10 @@ from openlit.guard.utils import (
     parse_llm_response,
     custom_rule_detection,
     guard_metrics,
-    guard_metric_attributes
+    guard_metric_attributes,
 )
 
+
 def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None) -> str:
     """
     Returns the system prompt used for LLM analysis, including custom categories if provided.
@@ -64,7 +65,9 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None) -> str
     """
 
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {value}" for key, value in custom_categories.items()])
+        custom_categories_str = "\n".join(
+            [f"- {key}: {value}" for key, value in custom_categories.items()]
+        )
         base_prompt += f"\n    Additional Categories:\n{custom_categories_str}"
 
     base_prompt += """
@@ -73,17 +76,25 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None) -> str
     """
     return base_prompt
 
+
 class PromptInjection:
     """Class to intialize Prompt Injection"""
 
-    def __init__(self, provider: Optional[str] = None, api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_rules: Optional[List[dict]] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 threshold_score: float = 0.25,
-                 collect_metrics: Optional[bool] = False):
+    def __init__(
+        self,
+        provider: Optional[str] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_rules: Optional[List[dict]] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        threshold_score: float = 0.25,
+        collect_metrics: Optional[bool] = False,
+    ):
         self.provider = provider
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.system_prompt = get_system_prompt(custom_categories)
         self.custom_rules = custom_rules or []
         self.threshold_score = threshold_score
@@ -93,11 +104,19 @@ class PromptInjection:
         """Functon to detect Prompt Injection and jailbreak attempts in input"""
 
         custom_rule_result = custom_rule_detection(text, self.custom_rules)
-        llm_result = JsonOutput(score=0, classification="none", explanation="none", verdict="none", guard="none")
+        llm_result = JsonOutput(
+            score=0,
+            classification="none",
+            explanation="none",
+            verdict="none",
+            guard="none",
+        )
 
         if self.provider:
             prompt = format_prompt(self.system_prompt, text)
-            llm_result = parse_llm_response(llm_response(self.provider, prompt, self.model, self.base_url))
+            llm_result = parse_llm_response(
+                llm_response(self.provider, prompt, self.model, self.base_url)
+            )
 
         result = max(custom_rule_result, llm_result, key=lambda x: x.score)
         score = 0 if result.classification == "none" else result.score
@@ -105,8 +124,9 @@ class PromptInjection:
 
         if self.collect_metrics is True:
             guard_counter = guard_metrics()
-            attributes = guard_metric_attributes(verdict, score, result.guard,
-                                                 result.classification, result.explanation)
+            attributes = guard_metric_attributes(
+                verdict, score, result.guard, result.classification, result.explanation
+            )
             guard_counter.add(1, attributes)
 
         return JsonOutput(
@@ -114,5 +134,5 @@ class PromptInjection:
             guard=result.guard,
             verdict=verdict,
             classification=result.classification,
-            explanation=result.explanation
+            explanation=result.explanation,
         )

openlit/guard/restrict_topic.py

@@ -11,10 +11,13 @@ from openlit.guard.utils import (
     llm_response,
     parse_llm_response,
     guard_metrics,
-    guard_metric_attributes
+    guard_metric_attributes,
 )
 
-def get_system_prompt(valid_topics: Optional[List[str]] = None, invalid_topics: Optional[List[str]] = None) -> str:
+
+def get_system_prompt(
+    valid_topics: Optional[List[str]] = None, invalid_topics: Optional[List[str]] = None
+) -> str:
     """
     Returns the system prompt used for LLM analysis, including valid and invalid topics if provided.
 
@@ -79,6 +82,7 @@ def get_system_prompt(valid_topics: Optional[List[str]] = None, invalid_topics:
 
     return base_prompt
 
+
 class TopicRestriction:
     """
     A class to validate if text belongs to valid or invalid topics using LLM.
@@ -92,12 +96,16 @@ class TopicRestriction:
         invalid_topics (Optional[List[str]]): List of invalid topics.
     """
 
-    def __init__(self, provider: Optional[str], valid_topics: Optional[List[str]] = None,
-                 api_key: Optional[str] = None, model: Optional[str] = None,
-                 base_url: Optional[str] = None,
-                 invalid_topics: Optional[List[str]] = None,
-                 collect_metrics: Optional[bool] = False,
-                ):
+    def __init__(
+        self,
+        provider: Optional[str],
+        valid_topics: Optional[List[str]] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        invalid_topics: Optional[List[str]] = None,
+        collect_metrics: Optional[bool] = False,
+    ):
         """
         Initializes the TopicRestriction with specified LLM settings and topics.
 
@@ -114,12 +122,18 @@ class TopicRestriction:
         """
         self.provider = provider
         if self.provider is None:
-            raise ValueError("An LLM provider must be specified for TopicRestriction Validator")
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+            raise ValueError(
+                "An LLM provider must be specified for TopicRestriction Validator"
+            )
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.system_prompt = get_system_prompt(valid_topics, invalid_topics)
         self.valid_topics = valid_topics
         if self.valid_topics is None:
-            raise ValueError("Valid Topics must be specified for TopicRestriction Validator")
+            raise ValueError(
+                "Valid Topics must be specified for TopicRestriction Validator"
+            )
         self.invalid_topics = invalid_topics or []
         self.collect_metrics = collect_metrics
 
@@ -139,14 +153,31 @@ class TopicRestriction:
 
         # Adjusted logic for consistency with updated JSON structure
         if llm_result.classification == "valid_topic":
-            result = JsonOutput(score=0, verdict="no", guard="topic_restriction", classification="valid_topic", explanation="Text fits into a valid topic.")
+            result = JsonOutput(
+                score=0,
+                verdict="no",
+                guard="topic_restriction",
+                classification="valid_topic",
+                explanation="Text fits into a valid topic.",
+            )
         else:
-            result = JsonOutput(score=1.0, verdict="yes", guard="topic_restriction", classification="invalid_topic", explanation="Text does not match any valid categories.")
+            result = JsonOutput(
+                score=1.0,
+                verdict="yes",
+                guard="topic_restriction",
+                classification="invalid_topic",
+                explanation="Text does not match any valid categories.",
+            )
 
         if self.collect_metrics:
             guard_counter = guard_metrics()
-            attributes = guard_metric_attributes(result.verdict, result.score, result.guard,
-                                                 result.classification, result.explanation)
+            attributes = guard_metric_attributes(
+                result.verdict,
+                result.score,
+                result.guard,
+                result.classification,
+                result.explanation,
+            )
             guard_counter.add(1, attributes)
 
         return result

openlit/guard/sensitive_topic.py

@@ -12,9 +12,10 @@ from openlit.guard.utils import (
     parse_llm_response,
     custom_rule_detection,
     guard_metrics,
-    guard_metric_attributes
+    guard_metric_attributes,
 )
 
+
 def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None) -> str:
     """
     Returns the system prompt used for LLM analysis, including custom categories if provided.
@@ -61,7 +62,9 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None) -> str
     """
 
     if custom_categories:
-        custom_categories_str = "\n".join([f"- {key}: {value}" for key, value in custom_categories.items()])
+        custom_categories_str = "\n".join(
+            [f"- {key}: {value}" for key, value in custom_categories.items()]
+        )
         base_prompt += f"\n    Additional Categories:\n{custom_categories_str}"
 
     base_prompt += """
@@ -71,6 +74,7 @@ def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None) -> str
 
     return base_prompt
 
+
 class SensitiveTopic:
     """
     A class to detect sensitive topics using LLM or custom rules.
@@ -84,14 +88,21 @@ class SensitiveTopic:
         custom_categories (Optional[Dict[str, str]]): Additional categories for sensitive topics.
     """
 
-    def __init__(self, provider: Optional[str] = None, api_key: Optional[str] = None,
-                 model: Optional[str] = None, base_url: Optional[str] = None,
-                 custom_rules: Optional[List[dict]] = None,
-                 custom_categories: Optional[Dict[str, str]] = None,
-                 threshold_score: float = 0.25,
-                 collect_metrics: Optional[bool] = False):
+    def __init__(
+        self,
+        provider: Optional[str] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        base_url: Optional[str] = None,
+        custom_rules: Optional[List[dict]] = None,
+        custom_categories: Optional[Dict[str, str]] = None,
+        threshold_score: float = 0.25,
+        collect_metrics: Optional[bool] = False,
+    ):
         self.provider = provider
-        self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)
+        self.api_key, self.model, self.base_url = setup_provider(
+            provider, api_key, model, base_url
+        )
         self.system_prompt = get_system_prompt(custom_categories)
         self.custom_rules = custom_rules or []
         self.threshold_score = threshold_score
@@ -101,11 +112,19 @@ class SensitiveTopic:
         """Function to detect sensitive topic in AI response"""
 
         custom_rule_result = custom_rule_detection(text, self.custom_rules)
-        llm_result = JsonOutput(score=0, classification="none", explanation="none", verdict="no", guard="none")
+        llm_result = JsonOutput(
+            score=0,
+            classification="none",
+            explanation="none",
+            verdict="no",
+            guard="none",
+        )
 
         if self.provider:
             prompt = format_prompt(self.system_prompt, text)
-            llm_result = parse_llm_response(llm_response(self.provider, prompt, self.model, self.base_url))
+            llm_result = parse_llm_response(
+                llm_response(self.provider, prompt, self.model, self.base_url)
+            )
 
         result = max(custom_rule_result, llm_result, key=lambda x: x.score)
         score = 0 if result.classification == "none" else result.score
@@ -113,8 +132,9 @@ class SensitiveTopic:
 
         if self.collect_metrics:
             guard_counter = guard_metrics()
-            attributes = guard_metric_attributes(verdict, score, result.guard,
-                                                 result.classification, result.explanation)
+            attributes = guard_metric_attributes(
+                verdict, score, result.guard, result.classification, result.explanation
+            )
             guard_counter.add(1, attributes)
 
         return JsonOutput(
@@ -122,5 +142,5 @@ class SensitiveTopic:
             guard=result.guard,
             verdict=verdict,
             classification=result.classification,
-            explanation=result.explanation
+            explanation=result.explanation,
         )