azure-ai-evaluation 1.0.0b3__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py

@@ -6,13 +6,14 @@ import copy
 import json
 import time
 import uuid
-from typing import Dict, List
+from typing import Any, Dict, List, Optional, cast
 
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client
 from azure.ai.evaluation._user_agent import USER_AGENT
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
 
+from .._model_tools._template_handler import TemplateParameters
 from .models import OpenAIChatCompletionsModel
 
 
@@ -33,7 +34,15 @@ class SimulationRequestDTO:
     :type template_parameters: Dict
     """
 
-    def __init__(self, url, headers, payload, params, templatekey, template_parameters):
+    def __init__(
+        self,
+        url: str,
+        headers: Dict[str, str],
+        payload: Dict[str, Any],
+        params: Dict[str, str],
+        templatekey: str,
+        template_parameters: Optional[TemplateParameters],
+    ):
         self.url = url
         self.headers = headers
         self.json = json.dumps(payload)
@@ -47,9 +56,12 @@ class SimulationRequestDTO:
         :return: The DTO as a dictionary.
         :rtype: Dict
         """
-        if self.templateParameters is not None:
-            self.templateParameters = {str(k): str(v) for k, v in self.templateParameters.items()}
-        return self.__dict__
+        toReturn = self.__dict__.copy()
+
+        if toReturn["templateParameters"] is not None:
+            toReturn["templateParameters"] = {str(k): str(v) for k, v in toReturn["templateParameters"].items()}
+
+        return toReturn
 
     def to_json(self):
         """Convert the DTO to a JSON string.
@@ -73,12 +85,12 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
     :keyword kwargs: Additional keyword arguments to pass to the parent class.
     """
 
-    def __init__(self, name: str, template_key: str, template_parameters, *args, **kwargs) -> None:
+    def __init__(self, name: str, template_key: str, template_parameters: TemplateParameters, **kwargs) -> None:
         self.tkey = template_key
         self.tparam = template_parameters
-        self.result_url = None
+        self.result_url: Optional[str] = None
 
-        super().__init__(name=name, *args, **kwargs)
+        super().__init__(name=name, **kwargs)
 
     def format_request_data(self, messages: List[Dict], **request_params) -> Dict:  # type: ignore[override]
         """Format the request data to query the model with.
@@ -160,7 +172,6 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         }
         # add all additional headers
         headers.update(self.additional_headers)  # type: ignore[arg-type]
-
         params = {}
         if self.api_version:
             params["api-version"] = self.api_version
@@ -184,8 +195,8 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
                 message=f"Received unexpected HTTP status: {response.status_code} {response.text()}", response=response
             )
 
-        response = response.json()
-        self.result_url = response["location"]
+        response_data = response.json()
+        self.result_url = cast(str, response_data["location"])
 
         retry_policy = AsyncRetryPolicy(  # set up retry configuration
             retry_on_status_codes=[202],  # on which statuses to retry
@@ -202,6 +213,12 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         time.sleep(15)
 
         async with get_async_http_client().with_policies(retry_policy=retry_policy) as exp_retry_client:
+            token = await self.token_manager.get_token_async()
+            proxy_headers = {
+                "Authorization": f"Bearer {token}",
+                "Content-Type": "application/json",
+                "User-Agent": USER_AGENT,
+            }
             response = await exp_retry_client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
                 self.result_url, headers=proxy_headers
             )

azure/ai/evaluation/simulator/_model_tools/_template_handler.py

@@ -2,25 +2,66 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Optional
+from typing import Dict, List, Optional, TypedDict, cast
+
+from typing_extensions import NotRequired
 
 from azure.ai.evaluation._model_configurations import AzureAIProject
 
 from ._rai_client import RAIClient
 
-CONTENT_HARM_TEMPLATES_COLLECTION_KEY = set(
-    [
-        "adv_qa",
-        "adv_conversation",
-        "adv_summarization",
-        "adv_search",
-        "adv_rewrite",
-        "adv_content_gen_ungrounded",
-        "adv_content_gen_grounded",
-        "adv_content_protected_material",
-        "adv_politics",
-    ]
-)
+CONTENT_HARM_TEMPLATES_COLLECTION_KEY = {
+    "adv_qa",
+    "adv_conversation",
+    "adv_summarization",
+    "adv_search",
+    "adv_rewrite",
+    "adv_content_gen_ungrounded",
+    "adv_content_gen_grounded",
+    "adv_content_protected_material",
+    "adv_politics",
+}
+
+
+class TemplateParameters(TypedDict):
+    """Parameters used in Templates
+
+    .. note::
+
+        This type is good enough to type check, but is incorrect. It's meant to represent a dictionary with a known
+        `metadata` key (Dict[str, str]), a known `ch_template_placeholder` key (str), and an unknown number of keys
+        that map to `str` values.
+
+        In typescript, this type would be spelled:
+
+        .. code-block:: typescript
+
+            type AdversarialTemplateParameters = {
+                [key: string]: string
+                ch_template_placeholder: string
+                metadata: {[index: string]: string} # Doesn't typecheck but gets the point across
+            }
+
+        At time of writing, this isn't possible to express with a TypedDict. TypedDicts must be "closed" in that
+        they fully specify all the keys they can contain.
+
+        `PEP 728 – TypedDict with Typed Extra Items <https://peps.python.org/pep-0728/>` is a proposal to support
+        this, but would only be available in Python 3.13 at the earliest.
+    """
+
+    metadata: Dict[str, str]
+    conversation_starter: str
+    ch_template_placeholder: str
+    group_of_people: NotRequired[str]
+    category: NotRequired[str]
+    target_population: NotRequired[str]
+    topic: NotRequired[str]
+
+
+class _CategorizedParameter(TypedDict):
+    parameters: List[TemplateParameters]
+    category: str
+    parameters_key: str
 
 
 class ContentHarmTemplatesUtils:
@@ -85,13 +126,19 @@ class AdversarialTemplate:
     :param template_parameters: The template parameters.
     """
 
-    def __init__(self, template_name, text, context_key, template_parameters=None) -> None:
+    def __init__(
+        self,
+        template_name: str,
+        text: Optional[str],
+        context_key: List,
+        template_parameters: Optional[List[TemplateParameters]] = None,
+    ) -> None:
         self.text = text
         self.context_key = context_key
         self.template_name = template_name
-        self.template_parameters = template_parameters
+        self.template_parameters = template_parameters or []
 
-    def __str__(self):
+    def __str__(self) -> str:
         return "{{ch_template_placeholder}}"
 
 
@@ -106,16 +153,13 @@ class AdversarialTemplateHandler:
     """
 
     def __init__(self, azure_ai_project: AzureAIProject, rai_client: RAIClient) -> None:
-        self.cached_templates_source = {}
-        # self.template_env = JinjaEnvironment(loader=JinjaFileSystemLoader(searchpath=template_dir))
         self.azure_ai_project = azure_ai_project
-        self.categorized_ch_parameters = None
+        self.categorized_ch_parameters: Optional[Dict[str, _CategorizedParameter]] = None
         self.rai_client = rai_client
 
-    async def _get_content_harm_template_collections(self, collection_key):
-
+    async def _get_content_harm_template_collections(self, collection_key: str) -> List[AdversarialTemplate]:
         if self.categorized_ch_parameters is None:
-            categorized_parameters = {}
+            categorized_parameters: Dict[str, _CategorizedParameter] = {}
             util = ContentHarmTemplatesUtils
 
             parameters = await self.rai_client.get_contentharm_parameters()
@@ -123,7 +167,7 @@ class AdversarialTemplateHandler:
             for k in parameters.keys():
                 template_key = util.get_template_key(k)
                 categorized_parameters[template_key] = {
-                    "parameters": parameters[k],
+                    "parameters": cast(List[TemplateParameters], parameters[k]),
                     "category": util.get_template_category(k),
                     "parameters_key": k,
                 }

azure/ai/evaluation/simulator/_model_tools/models.py

@@ -49,10 +49,10 @@ class LLMBase(ABC):
     Base class for all LLM models.
     """
 
-    def __init__(self, endpoint_url: str, name: str = "unknown", additional_headers: Optional[dict] = {}):
+    def __init__(self, endpoint_url: str, name: str = "unknown", additional_headers: Optional[Dict[str, str]] = None):
         self.endpoint_url = endpoint_url
         self.name = name
-        self.additional_headers = additional_headers
+        self.additional_headers = additional_headers or {}
         self.logger = logging.getLogger(repr(self))
 
         # Metric tracking
@@ -208,7 +208,7 @@ class OpenAICompletionsModel(LLMBase):
         *,
         endpoint_url: str,
         name: str = "OpenAICompletionsModel",
-        additional_headers: Optional[dict] = {},
+        additional_headers: Optional[Dict[str, str]] = None,
         api_version: Optional[str] = "2023-03-15-preview",
         token_manager: APITokenManager,
         azureml_model_deployment: Optional[str] = None,
@@ -220,7 +220,7 @@ class OpenAICompletionsModel(LLMBase):
         frequency_penalty: Optional[float] = 0,
         presence_penalty: Optional[float] = 0,
         stop: Optional[Union[List[str], str]] = None,
-        image_captions: Dict[str, str] = {},
+        image_captions: Optional[Dict[str, str]] = None,
         images_dir: Optional[str] = None,  # Note: unused, kept for class compatibility
     ):
         super().__init__(endpoint_url=endpoint_url, name=name, additional_headers=additional_headers)
@@ -234,7 +234,7 @@ class OpenAICompletionsModel(LLMBase):
         self.n = n
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
-        self.image_captions = image_captions
+        self.image_captions = image_captions or {}
 
         # Default stop to end token if not provided
         if not stop:
@@ -263,7 +263,7 @@ class OpenAICompletionsModel(LLMBase):
     def get_model_params(self):
         return {param: getattr(self, param) for param in self.model_param_names if getattr(self, param) is not None}
 
-    def format_request_data(self, prompt: str, **request_params) -> Dict[str, str]:
+    def format_request_data(self, prompt: Dict[str, str], **request_params) -> Dict[str, str]:  # type: ignore[override]
         """
         Format the request data for the OpenAI API.
         """
@@ -328,7 +328,7 @@ class OpenAICompletionsModel(LLMBase):
         # Format prompts and tag with index
         request_datas: List[Dict] = []
         for idx, prompt in enumerate(prompts):
-            prompt: Dict[str, str] = self.format_request_data(prompt, **request_params)
+            prompt = self.format_request_data(prompt, **request_params)
             prompt[self.prompt_idx_key] = idx  # type: ignore[assignment]
             request_datas.append(prompt)
 
@@ -447,7 +447,7 @@ class OpenAICompletionsModel(LLMBase):
 
         self._log_request(request_data)
 
-        token = await self.token_manager.get_token()
+        token = self.token_manager.get_token()
 
         headers = {
             "Content-Type": "application/json",
@@ -522,8 +522,8 @@ class OpenAIChatCompletionsModel(OpenAICompletionsModel):
     formats the prompt for chat completion.
     """
 
-    def __init__(self, name="OpenAIChatCompletionsModel", *args, **kwargs):
-        super().__init__(name=name, *args, **kwargs)
+    def __init__(self, name="OpenAIChatCompletionsModel", **kwargs):
+        super().__init__(name=name, **kwargs)
 
     def format_request_data(self, messages: List[dict], **request_params):  # type: ignore[override]
         request_data = {"messages": messages, **self.get_model_params()}

azure/ai/evaluation/simulator/_prompty/task_query_response.prompty

@@ -3,11 +3,6 @@ name: TaskSimulatorQueryResponse
 description: Gets queries and responses from a blob of text
 model:
   api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    api_key: ${env:AZURE_OPENAI_API_KEY}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     top_p: 1.0
@@ -41,8 +36,8 @@ On January 24, 1984, former Apple CEO Steve Jobs introduced the first Macintosh.
 Some years later, research firms IDC and Gartner reported that Apple's market share in the U.S. had increased to about 6%.
 <|text_end|>
 Output with 5 QnAs:
-[
-  {
+{
+  "qna":[{
     "q": "When did the former Apple CEO Steve Jobs introduced the first Macintosh?",
     "r": "January 24, 1984"
   },
@@ -61,8 +56,8 @@ Output with 5 QnAs:
   {
     "q": "What was the percentage increase of Apple's market share in the U.S., as reported by research firms IDC and Gartner?",
     "r": "6%"
-  }
-]
+  }]
+}
 Text:
 <|text_start|>
 {{ text }}

azure/ai/evaluation/simulator/_prompty/task_simulate.prompty

@@ -3,10 +3,6 @@ name: TaskSimulatorWithPersona
 description: Simulates a user to complete a conversation
 model:
   api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     top_p: 1.0
@@ -20,6 +16,9 @@ inputs:
     type: string
   conversation_history:
     type: dict
+  action:
+    type: string
+    default: continue the converasation and make sure the task is completed by asking relevant questions
 
 ---
 system:
@@ -29,8 +28,10 @@ Output must be in JSON format
 Here's a sample output:
 {
   "content": "Here is my follow-up question.",
-  "user": "user"
+  "role": "user"
 }
 
 Output with a json object that continues the conversation, given the conversation history:
 {{ conversation_history }}
+
+{{ action }}