azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/simulator/_model_tools/_identity_manager.py

@@ -3,20 +3,16 @@
 # ---------------------------------------------------------
 
 import asyncio
-import inspect
 import logging
 import os
 import time
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Optional, Union
+from typing import Dict, Optional, Union
 
-from azure.core.credentials import AccessToken, TokenCredential
 from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
 
-AZURE_TOKEN_REFRESH_INTERVAL = int(
-    os.getenv("AZURE_TOKEN_REFRESH_INTERVAL", "600")
-)  # token refresh interval in seconds
+AZURE_TOKEN_REFRESH_INTERVAL = 600  # seconds
 
 
 class TokenScope(Enum):
@@ -33,24 +29,24 @@ class APITokenManager(ABC):
     :param auth_header: Authorization header prefix. Defaults to "Bearer"
     :type auth_header: str
     :param credential: Azure credential object
-    :type credential: Optional[TokenCredential]
+    :type credential: Optional[Union[azure.identity.DefaultAzureCredential, azure.identity.ManagedIdentityCredential]
     """
 
     def __init__(
         self,
         logger: logging.Logger,
         auth_header: str = "Bearer",
-        credential: Optional[TokenCredential] = None,
+        credential: Optional[Union[DefaultAzureCredential, ManagedIdentityCredential]] = None,
     ) -> None:
         self.logger = logger
         self.auth_header = auth_header
-        self._lock: Optional[asyncio.Lock] = None
+        self._lock = None
         if credential is not None:
             self.credential = credential
         else:
             self.credential = self.get_aad_credential()
-        self.token: Optional[str] = None
-        self.last_refresh_time: Optional[float] = None
+        self.token = None
+        self.last_refresh_time = None
 
     @property
     def lock(self) -> asyncio.Lock:
@@ -77,26 +73,20 @@ class APITokenManager(ABC):
         identity_client_id = os.environ.get("DEFAULT_IDENTITY_CLIENT_ID", None)
         if identity_client_id is not None:
             self.logger.info(f"Using DEFAULT_IDENTITY_CLIENT_ID: {identity_client_id}")
-            return ManagedIdentityCredential(client_id=identity_client_id)
-
-        self.logger.info("Environment variable DEFAULT_IDENTITY_CLIENT_ID is not set, using DefaultAzureCredential")
-        return DefaultAzureCredential()
-
-    @abstractmethod
-    def get_token(self) -> str:
-        """Async method to get the API token. Subclasses should implement this method.
-
-        :return: API token
-        :rtype: str
-        """
+            credential = ManagedIdentityCredential(client_id=identity_client_id)
+        else:
+            self.logger.info("Environment variable DEFAULT_IDENTITY_CLIENT_ID is not set, using DefaultAzureCredential")
+            credential = DefaultAzureCredential()
+        return credential
 
     @abstractmethod
-    async def get_token_async(self) -> str:
+    async def get_token(self) -> str:
         """Async method to get the API token. Subclasses should implement this method.
 
         :return: API token
         :rtype: str
         """
+        pass  # pylint: disable=unnecessary-pass
 
 
 class ManagedIdentityAPITokenManager(APITokenManager):
@@ -110,18 +100,12 @@ class ManagedIdentityAPITokenManager(APITokenManager):
     :paramtype kwargs: Dict
     """
 
-    def __init__(
-        self,
-        token_scope: TokenScope,
-        logger: logging.Logger,
-        *,
-        auth_header: str = "Bearer",
-        credential: Optional[TokenCredential] = None,
-    ):
-        super().__init__(logger, auth_header=auth_header, credential=credential)
+    def __init__(self, token_scope: TokenScope, logger: logging.Logger, **kwargs: Dict):
+        super().__init__(logger, **kwargs)
         self.token_scope = token_scope
 
-    def get_token(self) -> str:
+    # Bug 3353724: This get_token is sync method, but it is defined as async method in the base class
+    def get_token(self) -> str:  # pylint: disable=invalid-overridden-method
         """Get the API token. If the token is not available or has expired, refresh the token.
 
         :return: API token
@@ -138,31 +122,6 @@ class ManagedIdentityAPITokenManager(APITokenManager):
 
         return self.token
 
-    async def get_token_async(self) -> str:
-        """Get the API token synchronously. If the token is not available or has expired, refresh it.
-
-        :return: API token
-        :rtype: str
-        """
-        if (
-            self.token is None
-            or self.last_refresh_time is None
-            or time.time() - self.last_refresh_time > AZURE_TOKEN_REFRESH_INTERVAL
-        ):
-            self.last_refresh_time = time.time()
-            get_token_method = self.credential.get_token(self.token_scope.value)
-            if inspect.isawaitable(get_token_method):
-                # If it's awaitable, await it
-                token_response: AccessToken = await get_token_method
-            else:
-                # Otherwise, call it synchronously
-                token_response = get_token_method
-
-            self.token = token_response.token
-            self.logger.info("Refreshed Azure endpoint token.")
-
-        return self.token
-
 
 class PlainTokenManager(APITokenManager):
     """Plain API Token Manager
@@ -175,18 +134,11 @@ class PlainTokenManager(APITokenManager):
     :paramtype kwargs: Dict
     """
 
-    def __init__(
-        self,
-        openapi_key: str,
-        logger: logging.Logger,
-        *,
-        auth_header: str = "Bearer",
-        credential: Optional[TokenCredential] = None,
-    ) -> None:
-        super().__init__(logger, auth_header=auth_header, credential=credential)
-        self.token: str = openapi_key
+    def __init__(self, openapi_key: str, logger: logging.Logger, **kwargs: Dict):
+        super().__init__(logger, **kwargs)
+        self.token = openapi_key
 
-    def get_token(self) -> str:
+    async def get_token(self) -> str:
         """Get the API token
 
         :return: API token

azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py

@@ -6,14 +6,14 @@ import copy
 import json
 import time
 import uuid
-from typing import Any, Dict, List, Optional, cast
+from typing import Dict, List
 
-from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client
-from azure.ai.evaluation._user_agent import USER_AGENT
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
 
-from .._model_tools._template_handler import TemplateParameters
+from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client
+from azure.ai.evaluation._user_agent import USER_AGENT
+
 from .models import OpenAIChatCompletionsModel
 
 
@@ -34,15 +34,7 @@ class SimulationRequestDTO:
     :type template_parameters: Dict
     """
 
-    def __init__(
-        self,
-        url: str,
-        headers: Dict[str, str],
-        payload: Dict[str, Any],
-        params: Dict[str, str],
-        templatekey: str,
-        template_parameters: Optional[TemplateParameters],
-    ):
+    def __init__(self, url, headers, payload, params, templatekey, template_parameters):
         self.url = url
         self.headers = headers
         self.json = json.dumps(payload)
@@ -56,12 +48,9 @@ class SimulationRequestDTO:
         :return: The DTO as a dictionary.
         :rtype: Dict
         """
-        toReturn = self.__dict__.copy()
-
-        if toReturn["templateParameters"] is not None:
-            toReturn["templateParameters"] = {str(k): str(v) for k, v in toReturn["templateParameters"].items()}
-
-        return toReturn
+        if self.templateParameters is not None:
+            self.templateParameters = {str(k): str(v) for k, v in self.templateParameters.items()}
+        return self.__dict__
 
     def to_json(self):
         """Convert the DTO to a JSON string.
@@ -85,12 +74,12 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
     :keyword kwargs: Additional keyword arguments to pass to the parent class.
     """
 
-    def __init__(self, name: str, template_key: str, template_parameters: TemplateParameters, **kwargs) -> None:
+    def __init__(self, name: str, template_key: str, template_parameters, *args, **kwargs) -> None:
         self.tkey = template_key
         self.tparam = template_parameters
-        self.result_url: Optional[str] = None
+        self.result_url = None
 
-        super().__init__(name=name, **kwargs)
+        super().__init__(name=name, *args, **kwargs)
 
     def format_request_data(self, messages: List[Dict], **request_params) -> Dict:  # type: ignore[override]
         """Format the request data to query the model with.
@@ -172,6 +161,7 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         }
         # add all additional headers
         headers.update(self.additional_headers)  # type: ignore[arg-type]
+
         params = {}
         if self.api_version:
             params["api-version"] = self.api_version
@@ -195,8 +185,8 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
                 message=f"Received unexpected HTTP status: {response.status_code} {response.text()}", response=response
             )
 
-        response_data = response.json()
-        self.result_url = cast(str, response_data["location"])
+        response = response.json()
+        self.result_url = response["location"]
 
         retry_policy = AsyncRetryPolicy(  # set up retry configuration
             retry_on_status_codes=[202],  # on which statuses to retry
@@ -213,12 +203,6 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         time.sleep(15)
 
         async with get_async_http_client().with_policies(retry_policy=retry_policy) as exp_retry_client:
-            token = await self.token_manager.get_token_async()
-            proxy_headers = {
-                "Authorization": f"Bearer {token}",
-                "Content-Type": "application/json",
-                "User-Agent": USER_AGENT,
-            }
             response = await exp_retry_client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
                 self.result_url, headers=proxy_headers
             )

azure/ai/evaluation/simulator/_model_tools/_rai_client.py

@@ -2,14 +2,15 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import os
-from typing import Any
+from typing import Any, Dict
 from urllib.parse import urljoin, urlparse
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
+
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client, get_http_client
-from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation._user_agent import USER_AGENT
-from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._model_configurations import AzureAIProject
 
 from ._identity_manager import APITokenManager
 
@@ -20,7 +21,7 @@ if "RAI_SVC_URL" in os.environ:
     print(f"Found RAI_SVC_URL in environment variable, using {api_url} for the service endpoint.")
 
 
-class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
+class RAIClient:
     """Client for the Responsible AI Service
 
     :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project
@@ -30,9 +31,7 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
     :type token_manage: ~azure.ai.evaluation.simulator._model_tools._identity_manager.APITokenManager
     """
 
-    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential,missing-client-constructor-parameter-kwargs
-        self, azure_ai_project: AzureAIProject, token_manager: APITokenManager
-    ) -> None:
+    def __init__(self, azure_ai_project: AzureAIProject, token_manager: APITokenManager) -> None:
         self.azure_ai_project = azure_ai_project
         self.token_manager = token_manager
 
@@ -74,18 +73,14 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
             timeout=5,
         )
         if response.status_code != 200:
-            msg = (
-                f"Failed to connect to your Azure AI project. Please check if the project scope is configured "
-                f"correctly, and make sure you have the necessary access permissions. "
-                f"Status code: {response.status_code}."
-            )
+            msg = f"Failed to retrieve the discovery service URL."
             raise EvaluationException(
                 message=msg,
+                internal_message=msg,
                 target=ErrorTarget.RAI_CLIENT,
-                category=ErrorCategory.PROJECT_ACCESS_ERROR,
-                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.SERVICE_UNAVAILABLE,
+                blame=ErrorBlame.UNKNOWN,
             )
-
         base_url = urlparse(response.json()["properties"]["discoveryUrl"])
         return f"{base_url.scheme}://{base_url.netloc}"
 
@@ -109,11 +104,7 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
         return self.contentharm_parameters
 
     async def get_jailbreaks_dataset(self, type: str) -> Any:
-        """Get the jailbreaks dataset, if exists
-
-        :param type: The dataset type. Should be one of 'xpia' or 'upia'
-        :type type: str
-        """
+        "Get the jailbreaks dataset, if exists"
         if self.jailbreaks_dataset is None:
             if type == "xpia":
                 self.jailbreaks_dataset = await self.get(self.xpia_jailbreaks_json_endpoint)
@@ -155,10 +146,8 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
         if response.status_code == 200:
             return response.json()
 
-        msg = (
-            "Azure safety evaluation service is not available in your current region, "
-            + "please go to https://aka.ms/azureaistudiosafetyeval to see which regions are supported"
-        )
+        msg = "Azure safety evaluation service is not available in your current region, "
+        "please go to https://aka.ms/azureaistudiosafetyeval to see which regions are supported"
         raise EvaluationException(
             message=msg,
             internal_message=msg,

azure/ai/evaluation/simulator/_model_tools/_template_handler.py

@@ -2,66 +2,25 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Dict, List, Optional, TypedDict, cast
-
-from typing_extensions import NotRequired
+from typing import Any, Dict, Optional
 
 from azure.ai.evaluation._model_configurations import AzureAIProject
 
 from ._rai_client import RAIClient
 
-CONTENT_HARM_TEMPLATES_COLLECTION_KEY = {
-    "adv_qa",
-    "adv_conversation",
-    "adv_summarization",
-    "adv_search",
-    "adv_rewrite",
-    "adv_content_gen_ungrounded",
-    "adv_content_gen_grounded",
-    "adv_content_protected_material",
-    "adv_politics",
-}
-
-
-class TemplateParameters(TypedDict):
-    """Parameters used in Templates
-
-    .. note::
-
-        This type is good enough to type check, but is incorrect. It's meant to represent a dictionary with a known
-        `metadata` key (Dict[str, str]), a known `ch_template_placeholder` key (str), and an unknown number of keys
-        that map to `str` values.
-
-        In typescript, this type would be spelled:
-
-        .. code-block:: typescript
-
-            type AdversarialTemplateParameters = {
-                [key: string]: string
-                ch_template_placeholder: string
-                metadata: {[index: string]: string} # Doesn't typecheck but gets the point across
-            }
-
-        At time of writing, this isn't possible to express with a TypedDict. TypedDicts must be "closed" in that
-        they fully specify all the keys they can contain.
-
-        `PEP 728 – TypedDict with Typed Extra Items <https://peps.python.org/pep-0728/>` is a proposal to support
-        this, but would only be available in Python 3.13 at the earliest.
-    """
-
-    metadata: Dict[str, str]
-    conversation_starter: str
-    ch_template_placeholder: str
-    group_of_people: NotRequired[str]
-    category: NotRequired[str]
-    target_population: NotRequired[str]
-    topic: NotRequired[str]
-
-
-class _CategorizedParameter(TypedDict):
-    parameters: List[TemplateParameters]
-    category: str
-    parameters_key: str
+CONTENT_HARM_TEMPLATES_COLLECTION_KEY = set(
+    [
+        "adv_qa",
+        "adv_conversation",
+        "adv_summarization",
+        "adv_search",
+        "adv_rewrite",
+        "adv_content_gen_ungrounded",
+        "adv_content_gen_grounded",
+        "adv_content_protected_material",
+        "adv_politics",
+    ]
+)
 
 
 class ContentHarmTemplatesUtils:
@@ -126,19 +85,13 @@ class AdversarialTemplate:
     :param template_parameters: The template parameters.
     """
 
-    def __init__(
-        self,
-        template_name: str,
-        text: Optional[str],
-        context_key: List,
-        template_parameters: Optional[List[TemplateParameters]] = None,
-    ) -> None:
+    def __init__(self, template_name, text, context_key, template_parameters=None) -> None:
         self.text = text
         self.context_key = context_key
         self.template_name = template_name
-        self.template_parameters = template_parameters or []
+        self.template_parameters = template_parameters
 
-    def __str__(self) -> str:
+    def __str__(self):
         return "{{ch_template_placeholder}}"
 
 
@@ -153,13 +106,16 @@ class AdversarialTemplateHandler:
     """
 
     def __init__(self, azure_ai_project: AzureAIProject, rai_client: RAIClient) -> None:
+        self.cached_templates_source = {}
+        # self.template_env = JinjaEnvironment(loader=JinjaFileSystemLoader(searchpath=template_dir))
         self.azure_ai_project = azure_ai_project
-        self.categorized_ch_parameters: Optional[Dict[str, _CategorizedParameter]] = None
+        self.categorized_ch_parameters = None
         self.rai_client = rai_client
 
-    async def _get_content_harm_template_collections(self, collection_key: str) -> List[AdversarialTemplate]:
+    async def _get_content_harm_template_collections(self, collection_key):
+
         if self.categorized_ch_parameters is None:
-            categorized_parameters: Dict[str, _CategorizedParameter] = {}
+            categorized_parameters = {}
             util = ContentHarmTemplatesUtils
 
             parameters = await self.rai_client.get_contentharm_parameters()
@@ -167,7 +123,7 @@ class AdversarialTemplateHandler:
             for k in parameters.keys():
                 template_key = util.get_template_key(k)
                 categorized_parameters[template_key] = {
-                    "parameters": cast(List[TemplateParameters], parameters[k]),
+                    "parameters": parameters[k],
                     "category": util.get_template_category(k),
                     "parameters_key": k,
                 }

azure/ai/evaluation/simulator/_model_tools/models.py

@@ -12,9 +12,10 @@ from abc import ABC, abstractmethod
 from collections import deque
 from typing import Deque, Dict, List, Optional, Union
 from urllib.parse import urlparse
+import ast
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 from ._identity_manager import APITokenManager
 
@@ -28,15 +29,16 @@ def get_model_class_from_url(endpoint_url: str):
 
     if endpoint_path.endswith("chat/completions"):
         return OpenAIChatCompletionsModel
-    if endpoint_path.endswith("completions"):
+    elif endpoint_path.endswith("completions"):
         return OpenAICompletionsModel
-    raise EvaluationException(
-        message=f"Unknown API type for endpoint {endpoint_url}",
-        internal_message="Unknown API type",
-        error_category=ErrorCategory.UNKNOWN_FIELD,
-        error_blame=ErrorBlame.USER_ERROR,
-        error_target=ErrorTarget.MODELS,
-    )
+    else:
+        raise EvaluationException(
+            message=f"Unknown API type for endpoint {endpoint_url}",
+            internal_message="Unknown API type",
+            error_category=ErrorCategory.UNKNOWN_FIELD,
+            error_blame=ErrorBlame.USER_ERROR,
+            error_target=ErrorTarget.MODELS,
+        )
 
 
 # ===========================================================
@@ -49,10 +51,10 @@ class LLMBase(ABC):
     Base class for all LLM models.
     """
 
-    def __init__(self, endpoint_url: str, name: str = "unknown", additional_headers: Optional[Dict[str, str]] = None):
+    def __init__(self, endpoint_url: str, name: str = "unknown", additional_headers: Optional[dict] = {}):
         self.endpoint_url = endpoint_url
         self.name = name
-        self.additional_headers = additional_headers or {}
+        self.additional_headers = additional_headers
         self.logger = logging.getLogger(repr(self))
 
         # Metric tracking
@@ -208,7 +210,7 @@ class OpenAICompletionsModel(LLMBase):
         *,
         endpoint_url: str,
         name: str = "OpenAICompletionsModel",
-        additional_headers: Optional[Dict[str, str]] = None,
+        additional_headers: Optional[dict] = {},
         api_version: Optional[str] = "2023-03-15-preview",
         token_manager: APITokenManager,
         azureml_model_deployment: Optional[str] = None,
@@ -220,7 +222,7 @@ class OpenAICompletionsModel(LLMBase):
         frequency_penalty: Optional[float] = 0,
         presence_penalty: Optional[float] = 0,
         stop: Optional[Union[List[str], str]] = None,
-        image_captions: Optional[Dict[str, str]] = None,
+        image_captions: Dict[str, str] = {},
         images_dir: Optional[str] = None,  # Note: unused, kept for class compatibility
     ):
         super().__init__(endpoint_url=endpoint_url, name=name, additional_headers=additional_headers)
@@ -234,7 +236,7 @@ class OpenAICompletionsModel(LLMBase):
         self.n = n
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
-        self.image_captions = image_captions or {}
+        self.image_captions = image_captions
 
         # Default stop to end token if not provided
         if not stop:
@@ -263,7 +265,7 @@ class OpenAICompletionsModel(LLMBase):
     def get_model_params(self):
         return {param: getattr(self, param) for param in self.model_param_names if getattr(self, param) is not None}
 
-    def format_request_data(self, prompt: Dict[str, str], **request_params) -> Dict[str, str]:  # type: ignore[override]
+    def format_request_data(self, prompt: str, **request_params) -> Dict[str, str]:
         """
         Format the request data for the OpenAI API.
         """
@@ -328,7 +330,7 @@ class OpenAICompletionsModel(LLMBase):
         # Format prompts and tag with index
         request_datas: List[Dict] = []
         for idx, prompt in enumerate(prompts):
-            prompt = self.format_request_data(prompt, **request_params)
+            prompt: Dict[str, str] = self.format_request_data(prompt, **request_params)
             prompt[self.prompt_idx_key] = idx  # type: ignore[assignment]
             request_datas.append(prompt)
 
@@ -447,7 +449,7 @@ class OpenAICompletionsModel(LLMBase):
 
         self._log_request(request_data)
 
-        token = self.token_manager.get_token()
+        token = await self.token_manager.get_token()
 
         headers = {
             "Content-Type": "application/json",
@@ -522,8 +524,8 @@ class OpenAIChatCompletionsModel(OpenAICompletionsModel):
     formats the prompt for chat completion.
     """
 
-    def __init__(self, name="OpenAIChatCompletionsModel", **kwargs):
-        super().__init__(name=name, **kwargs)
+    def __init__(self, name="OpenAIChatCompletionsModel", *args, **kwargs):
+        super().__init__(name=name, *args, **kwargs)
 
     def format_request_data(self, messages: List[dict], **request_params):  # type: ignore[override]
         request_data = {"messages": messages, **self.get_model_params()}

azure/ai/evaluation/simulator/_prompty/task_query_response.prompty

@@ -3,6 +3,11 @@ name: TaskSimulatorQueryResponse
 description: Gets queries and responses from a blob of text
 model:
   api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     top_p: 1.0
@@ -28,16 +33,15 @@ Answer must not be more than 5 words
 Answer must be picked from Text as is
 Question should be as descriptive as possible and must include as much context as possible from Text
 Output must always have the provided number of QnAs
-Output must be in JSON format.
-Output must have {{num_queries}} objects in the format specified below. Any other count is unacceptable.
+Output must be in JSON format
 Text:
 <|text_start|>
 On January 24, 1984, former Apple CEO Steve Jobs introduced the first Macintosh. In late 2003, Apple had 2.06 percent of the desktop share in the United States.
 Some years later, research firms IDC and Gartner reported that Apple's market share in the U.S. had increased to about 6%.
 <|text_end|>
 Output with 5 QnAs:
-{
-  "qna":[{
+[
+  {
     "q": "When did the former Apple CEO Steve Jobs introduced the first Macintosh?",
     "r": "January 24, 1984"
   },
@@ -56,8 +60,8 @@ Output with 5 QnAs:
   {
     "q": "What was the percentage increase of Apple's market share in the U.S., as reported by research firms IDC and Gartner?",
     "r": "6%"
-  }]
-}
+  }
+]
 Text:
 <|text_start|>
 {{ text }}

azure/ai/evaluation/simulator/_prompty/task_simulate.prompty

@@ -3,6 +3,10 @@ name: TaskSimulatorWithPersona
 description: Simulates a user to complete a conversation
 model:
   api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     top_p: 1.0
@@ -16,9 +20,6 @@ inputs:
     type: string
   conversation_history:
     type: dict
-  action:
-    type: string
-    default: continue the converasation and make sure the task is completed by asking relevant questions
 
 ---
 system:
@@ -28,10 +29,8 @@ Output must be in JSON format
 Here's a sample output:
 {
   "content": "Here is my follow-up question.",
-  "role": "user"
+  "user": "user"
 }
 
 Output with a json object that continues the conversation, given the conversation history:
 {{ conversation_history }}
-
-{{ action }}