azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure_ai_evaluation-1.0.0b1.dist-info/METADATA

@@ -0,0 +1,377 @@
+Metadata-Version: 2.1
+Name: azure-ai-evaluation
+Version: 1.0.0b1
+Summary: Microsoft Azure Evaluation Library for Python
+Home-page: https://github.com/Azure/azure-sdk-for-python
+Author: Microsoft Corporation
+Author-email: azuresdkengsysadmins@microsoft.com
+License: MIT License
+Project-URL: Bug Reports, https://github.com/Azure/azure-sdk-for-python/issues
+Project-URL: Source, https://github.com/Azure/azure-sdk-for-python
+Keywords: azure,azure sdk
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: promptflow-devkit >=1.15.0
+Requires-Dist: promptflow-core >=1.15.0
+Requires-Dist: pyjwt >=2.8.0
+Requires-Dist: azure-identity >=1.12.0
+Requires-Dist: azure-core >=1.30.2
+Requires-Dist: nltk >=3.9.1
+Requires-Dist: rouge-score >=0.1.2
+Requires-Dist: numpy >=1.23.2 ; python_version < "3.12"
+Requires-Dist: numpy >=1.26.4 ; python_version >= "3.12"
+Provides-Extra: pf-azure
+Requires-Dist: promptflow-azure <2.0.0,>=1.15.0 ; extra == 'pf-azure'
+
+# Azure AI Evaluation client library for Python
+
+## Getting started
+
+### Install the package
+
+Install the Azure AI Evaluation library for Python with:
+
+```bash
+pip install azure-ai-evaluation
+```
+
+## Key concepts
+
+Evaluators are custom or prebuilt classes or functions that are designed to measure the quality of the outputs from language models.
+
+## Examples
+
+Users can create evaluator runs on the local machine as shown in the example below:
+
+```python
+import os
+from pprint import pprint
+
+from azure.ai.evaluation import evaluate, RelevanceEvaluator, ViolenceEvaluator
+
+
+def response_length(response, **kwargs):
+    return {"value": len(response)}
+
+
+if __name__ == "__main__":
+    # Built-in evaluators
+    # Initialize Azure OpenAI Model Configuration
+    model_config = {
+        "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"),
+        "api_key": os.environ.get("AZURE_OPENAI_KEY"),
+        "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
+    }
+
+    # Initialzing Relevance Evaluator
+    relevance_eval = RelevanceEvaluator(model_config)
+
+    # Running Relevance Evaluator on single input row
+    relevance_score = relevance_eval(
+        response="The Alpine Explorer Tent is the most waterproof.",
+        context="From the our product list,"
+        " the alpine explorer tent is the most waterproof."
+        " The Adventure Dining Table has higher weight.",
+        query="Which tent is the most waterproof?",
+    )
+
+    pprint(relevance_score)
+    # {'gpt_relevance': 5.0}
+
+    # Content Safety Evaluator
+
+    # Initialize Project Scope
+    azure_ai_project = {
+        "subscription_id": "e0fd569c-e34a-4249-8c24-e8d723c7f054",
+        "resource_group_name": "rg-test",
+        "project_name": "project-test",
+    }
+
+    violence_eval = ViolenceEvaluator(azure_ai_project)
+    violence_score = violence_eval(query="What is the capital of France?", response="Paris.")
+    pprint(violence_score)
+    # {'violence': 'Very low',
+    # 'violence_reason': "The system's response is a straightforward factual response "
+    #                    'to a geography question. There is no violent content or '
+    #                    'language present.',
+    # 'violence_score': 0}
+
+    # Code based evaluator
+    response_length("The Alpine Explorer Tent is the most waterproof.")
+    # {'value': 48}
+
+    # Using multiple evaluators together using `Evaluate` API
+
+    result = evaluate(
+        data="evaluate_test_data.jsonl",
+        evaluators={
+            "response_length": response_length,
+            "violence": violence_eval,
+        },
+    )
+
+    pprint(result)
+```
+## Simulator
+
+Sample application prompty
+
+```yaml
+---
+name: ApplicationPrompty
+description: Simulates an application
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.0
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: text
+
+inputs:
+  conversation_history:
+    type: dict
+
+---
+system:
+You are a helpful assistant and you're helping with the user's query. Keep the conversation engaging and interesting.
+
+Output with a string that continues the conversation, responding to the latest message from the user, given the conversation history:
+{{ conversation_history }}
+
+```
+Application code:
+
+```python
+import json
+import asyncio
+from typing import Any, Dict, List, Optional
+from azure.ai.evaluation.synthetic import Simulator
+from promptflow.client import load_flow
+from azure.identity import DefaultAzureCredential
+import os
+
+azure_ai_project = {
+    "subscription_id": os.environ.get("AZURE_SUBSCRIPTION_ID"),
+    "resource_group_name": os.environ.get("RESOURCE_GROUP"),
+    "project_name": os.environ.get("PROJECT_NAME"),
+    "credential": DefaultAzureCredential(),
+}
+
+import wikipedia
+wiki_search_term = "Leonardo da vinci"
+wiki_title = wikipedia.search(wiki_search_term)[0]
+wiki_page = wikipedia.page(wiki_title)
+text = wiki_page.summary[:1000]
+
+def method_to_invoke_application_prompty(query: str):
+    try:
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, "application.prompty")
+        _flow = load_flow(source=prompty_path, model={
+            "configuration": azure_ai_project
+        })
+        response = _flow(
+            query=query,
+            context=context,
+            conversation_history=messages_list
+        )
+        return response
+    except:
+        print("Something went wrong invoking the prompty")
+        return "something went wrong"
+
+async def callback(
+    messages: List[Dict],
+    stream: bool = False,
+    session_state: Any = None,  # noqa: ANN401
+    context: Optional[Dict[str, Any]] = None,
+) -> dict:
+    messages_list = messages["messages"]
+    # get last message
+    latest_message = messages_list[-1]
+    query = latest_message["content"]
+    context = None
+    # call your endpoint or ai application here
+    response = method_to_invoke_application_prompty(query)
+    # we are formatting the response to follow the openAI chat protocol format
+    formatted_response = {
+        "content": response,
+        "role": "assistant",
+        "context": {
+            "citations": None,
+        },
+    }
+    messages["messages"].append(formatted_response)
+    return {"messages": messages["messages"], "stream": stream, "session_state": session_state, "context": context}
+
+
+
+async def main():
+    simulator = Simulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
+    outputs = await simulator(
+        target=callback,
+        text=text,
+        num_queries=2,
+        max_conversation_turns=4,
+        user_persona=[
+            f"I am a student and I want to learn more about {wiki_search_term}",
+            f"I am a teacher and I want to teach my students about {wiki_search_term}"
+        ],
+    )
+    print(json.dumps(outputs))
+
+if __name__ == "__main__":
+    os.environ["AZURE_SUBSCRIPTION_ID"] = ""
+    os.environ["RESOURCE_GROUP"] = ""
+    os.environ["PROJECT_NAME"] = ""
+    os.environ["AZURE_OPENAI_API_KEY"] = ""
+    os.environ["AZURE_OPENAI_ENDPOINT"] = ""
+    os.environ["AZURE_DEPLOYMENT"] = ""
+    asyncio.run(main())
+    print("done!")
+```
+
+Simulators allow users to generate synthentic data using their application. Simulator expects the user to have a callback method that invokes
+their AI application. Here's a sample of a callback which invokes AsyncAzureOpenAI:
+
+```python
+from from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario
+from azure.identity import DefaultAzureCredential
+from typing import Any, Dict, List, Optional
+import asyncio
+
+
+azure_ai_project = {
+    "subscription_id": <subscription_id>,
+    "resource_group_name": <resource_group_name>,
+    "project_name": <project_name>
+}
+
+async def callback(
+    messages: List[Dict],
+    stream: bool = False,
+    session_state: Any = None,
+    context: Dict[str, Any] = None
+) -> dict:
+    messages_list = messages["messages"]
+    # get last message
+    latest_message = messages_list[-1]
+    query = latest_message["content"]
+    context = None
+    if 'file_content' in messages["template_parameters"]:
+        query += messages["template_parameters"]['file_content']
+    # the next few lines explains how to use the AsyncAzureOpenAI's chat.completions
+    # to respond to the simulator. You should replace it with a call to your model/endpoint/application
+    # make sure you pass the `query` and format the response as we have shown below
+    from openai import AsyncAzureOpenAI
+    oai_client = AsyncAzureOpenAI(
+        api_key=<api_key>,
+        azure_endpoint=<endpoint>,
+        api_version="2023-12-01-preview",
+    )
+    try:
+        response_from_oai_chat_completions = await oai_client.chat.completions.create(messages=[{"content": query, "role": "user"}], model="gpt-4", max_tokens=300)
+    except Exception as e:
+        print(f"Error: {e}")
+        # to continue the conversation, return the messages, else you can fail the adversarial with an exception
+        message = {
+            "content": "Something went wrong. Check the exception e for more details.",
+            "role": "assistant",
+            "context": None,
+        }
+        messages["messages"].append(message)
+        return {
+            "messages": messages["messages"],
+            "stream": stream,
+            "session_state": session_state
+        }
+    response_result = response_from_oai_chat_completions.choices[0].message.content
+    formatted_response = {
+        "content": response_result,
+        "role": "assistant",
+        "context": {},
+    }
+    messages["messages"].append(formatted_response)
+    return {
+        "messages": messages["messages"],
+        "stream": stream,
+        "session_state": session_state,
+        "context": context
+    }
+
+```
+### Adversarial QA:
+```python
+scenario = AdversarialScenario.ADVERSARIAL_QA
+simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
+
+outputs = asyncio.run(
+    simulator(
+        scenario=scenario,
+        max_conversation_turns=1,
+        max_simulation_results=3,
+        target=callback
+    )
+)
+
+print(outputs.to_eval_qa_json_lines())
+```
+### Direct Attack Simulator
+
+```python
+scenario = AdversarialScenario.ADVERSARIAL_QA
+simulator = DirectAttackSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
+
+outputs = asyncio.run(
+    simulator(
+        scenario=scenario,
+        max_conversation_turns=1,
+        max_simulation_results=2,
+        target=callback
+    )
+)
+
+print(outputs)
+```
+## Troubleshooting
+
+## Next steps
+
+## Contributing
+
+
+# Release History
+
+## 1.0.0b1 (2024-09-20)
+
+### Breaking Changes
+
+- The `synthetic` namespace has been renamed to `simulator`, and sub-namespaces under this module have been removed
+- The `evaluate` and `evaluators` namespaces have been removed, and everything previously exposed in those modules has been added to the root namespace `azure.ai.evaluation`  
+- The parameter name `project_scope` in content safety evaluators have been renamed to `azure_ai_project` for consistency with evaluate API and simulators.
+- Model configurations classes are now of type `TypedDict` and are exposed in the `azure.ai.evaluation` module instead of coming from `promptflow.core`.  
+- Updated the parameter names for `question` and `answer` in built-in evaluators to more generic terms: `query` and `response`.
+
+### Features Added
+
+- First preview
+- This package is port of `promptflow-evals`. New features will be added only to this package moving forward.
+- Added a `TypedDict` for `AzureAIProject` that allows for better intellisense and type checking when passing in project information

azure_ai_evaluation-1.0.0b1.dist-info/RECORD

@@ -0,0 +1,97 @@
+azure/ai/evaluation/__init__.py,sha256=aCQXu_6B3tB-WSRtdPrMUODYMytwK1pVoOTwkR5EItg,1984
+azure/ai/evaluation/_constants.py,sha256=MtXK9FV3TgDiq8IYSMkSDbNXVQsZw62D26g7ZXJ62NU,1422
+azure/ai/evaluation/_exceptions.py,sha256=HUMfvguDc7ygcbs3MTK14R3PK7UxGNWQQHH3hYXIV3U,4168
+azure/ai/evaluation/_http_utils.py,sha256=13G64fWveezBdcmjwwwbJrIkdZp2ns3Jvd-addKw1SU,13983
+azure/ai/evaluation/_model_configurations.py,sha256=D02AzOdyO6LQCia0k232Msd7ro35-EcwmlQ0tOD_5H0,652
+azure/ai/evaluation/_user_agent.py,sha256=O2y-QPBAcw7w7qQ6M2aRPC3Vy3TKd789u5lcs2yuFaI,290
+azure/ai/evaluation/_version.py,sha256=oZ93SfxVsREL_UaqV9_yCzfIy_LG5t9AwK_uEE8HjXI,201
+azure/ai/evaluation/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+azure/ai/evaluation/_common/__init__.py,sha256=LHTkf6dMLLxikrGNgbUuREBVQcs4ORHR6Eryo4bm9M8,586
+azure/ai/evaluation/_common/constants.py,sha256=ZKHGVgGA1Fc6Pvo22X-CeOUX6-m0q_UwpOKOWATTSuI,1639
+azure/ai/evaluation/_common/rai_service.py,sha256=Cci_YnX1XeYHiZX4DTVj6_bAj2JADXh1znXS_cDM45M,17540
+azure/ai/evaluation/_common/utils.py,sha256=Ed2pgAXzswrOskWopYyRGdPMOkwbP-cgj32kBOuWMZw,2941
+azure/ai/evaluation/_evaluate/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
+azure/ai/evaluation/_evaluate/_eval_run.py,sha256=eUUUzF6o3FjkQ6L5aZnESQjWM_UscM5MnCD-5P0dldA,20237
+azure/ai/evaluation/_evaluate/_evaluate.py,sha256=HP7J39YUn9yMKLl2z-k-7_BVpzQIhDQwFOExQk6nb9w,28379
+azure/ai/evaluation/_evaluate/_utils.py,sha256=-8xv0TF3QQPfZlEpLOCGSq2Nvb-esZfpaDySOHLeey4,9680
+azure/ai/evaluation/_evaluate/_batch_run_client/__init__.py,sha256=BkxhojWca3e2QM3hFwO2xrLiiQ0i-3f8wsMfOx1zchs,361
+azure/ai/evaluation/_evaluate/_batch_run_client/batch_run_context.py,sha256=wL_gNM2u8_DYUKopxFnJFc0cUAxWo9HV5gNSfZoluJo,2983
+azure/ai/evaluation/_evaluate/_batch_run_client/code_client.py,sha256=xmck4Wlp0wrzkN6sZsmW-7MMAc1rIw16xwSV52_V8to,7120
+azure/ai/evaluation/_evaluate/_batch_run_client/proxy_client.py,sha256=_2NynIyH_QFZQf5TDZ9x_FtDUXrzvMfGVuTtpZJazpo,1981
+azure/ai/evaluation/_evaluate/_telemetry/__init__.py,sha256=gF0mHhCYjrlOBgR4I6mQ5KIfExpq7W2On31ITmjYN8M,6400
+azure/ai/evaluation/_evaluators/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
+azure/ai/evaluation/_evaluators/_bleu/__init__.py,sha256=quKKO0kvOSkky5hcoNBvgBuMeeVRFCE9GSv70mAdGP4,260
+azure/ai/evaluation/_evaluators/_bleu/_bleu.py,sha256=q8hTVC8Ubfb0lQmlGnsuEb6J7Qx5c7eYso4ss6RhgOw,2413
+azure/ai/evaluation/_evaluators/_chat/__init__.py,sha256=xOsSHYNGJJiZvBMPbmLd_-ZZs8_15Sblvk-OF7iVoIo,250
+azure/ai/evaluation/_evaluators/_chat/_chat.py,sha256=GsK7EERUNo5uuh025EazxpK6wuN7mzz891_l8WFxqXg,14692
+azure/ai/evaluation/_evaluators/_chat/retrieval/__init__.py,sha256=DmBjBkwDDlCsSGpBeXfpfMM9ekxIJs62dij4rBXND7k,273
+azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py,sha256=X_2UzFd-FP032nT4fjH1EY67bvIcRpmmjv9KQrXpfUE,5739
+azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty,sha256=NoHNDf_UE8BGAixqZPhRQ4ecxOUi9w9gO8HfHhJuxaY,1761
+azure/ai/evaluation/_evaluators/_coherence/__init__.py,sha256=GRqcSCQse02Spyki0UsRNWMIXiea2lLtPPXNGvkJzQ0,258
+azure/ai/evaluation/_evaluators/_coherence/_coherence.py,sha256=sOP2rYwGe1k_80FhBr5PGG8metmpJwGfx4U7-aVroIs,4121
+azure/ai/evaluation/_evaluators/_coherence/coherence.prompty,sha256=WVEXxKmh_Gbb11_00N2WCIIJSMgPssFxJ5h2--rMG-w,2725
+azure/ai/evaluation/_evaluators/_content_safety/__init__.py,sha256=mR5CbcMyxV9GQoY71Saoi0bQTpEB74HrYmM8gcVhnAg,746
+azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py,sha256=VUazS5AcJ5Z-BOrqfCRt5SdRhmTlxWdxl03FML0zHq4,3998
+azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py,sha256=Yb2x-sI5_GivAe8o8BvA2gM64yQCv73v5LS_QyiR9vQ,2802
+azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py,sha256=7cLTnbz0iHderZeXPKYbbBSwvKYZtMiExEMYQSfggd4,11909
+azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py,sha256=Ur1gQPBdr0b_Ov8K0Jflmvua2mgSrk9DhlPOAQR-7w4,2875
+azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py,sha256=MrCOPNmv6PAkVFKDrwNfXeOyhTmyDa2Tik_xEFKxYwg,2676
+azure/ai/evaluation/_evaluators/_content_safety/_sexual.py,sha256=MyRI9QtqElb2gDGPGnyQYfAHsjdw_oK6Ekjz4Voo9rg,2644
+azure/ai/evaluation/_evaluators/_content_safety/_violence.py,sha256=YH0NxossokGX5tFKAIZJvMiprs34KuiNQnMXAbOOMCM,2668
+azure/ai/evaluation/_evaluators/_eci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+azure/ai/evaluation/_evaluators/_eci/_eci.py,sha256=vm8G-YvZuXipvJSixA6mktEL7xQvqnsHcD1trqh7Cxw,4058
+azure/ai/evaluation/_evaluators/_f1_score/__init__.py,sha256=aEVbO7iMoF20obdpLQKcKm69Yyu3mYnblKELLqu8OGI,260
+azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py,sha256=KNMni1m8dtaC6kLf9JRUVfJ1hZfYBi-g6iZiruJ_Hyc,4553
+azure/ai/evaluation/_evaluators/_fluency/__init__.py,sha256=EEJw39xRa0bOAA1rELTTKXQu2s60n_7CZQRD0Gu2QVw,259
+azure/ai/evaluation/_evaluators/_fluency/_fluency.py,sha256=0KSSYQjWzPpGLBK-NIkIdfp0fPcCOpjIkuKMbYprqI0,4082
+azure/ai/evaluation/_evaluators/_fluency/fluency.prompty,sha256=RparSdDZs-xiGbq7lRifz9z7jaD10ldXDU3E7sO0v2s,2579
+azure/ai/evaluation/_evaluators/_gleu/__init__.py,sha256=Ae2EvQ7gqiYAoNO3LwGIhdAAjJPJDfT85rQGKrRrmbA,260
+azure/ai/evaluation/_evaluators/_gleu/_gleu.py,sha256=qQR5Ic4K4biHrMm8XLe1DtKc3MSwbfCjK_-CLpL9aCc,2326
+azure/ai/evaluation/_evaluators/_groundedness/__init__.py,sha256=UYNJUeRvBwcSVFyZpdsf29un5eyaDzYoo3QvC1gvlLg,274
+azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py,sha256=BZdr2Gqgos1CDTnMPJPUfoa_Pkq9WxgdL-Qem6GbINs,4301
+azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty,sha256=dDclp_YowBjeiGhwmCxCnS4A3K9r4v2tzsUm-ccLt-I,3199
+azure/ai/evaluation/_evaluators/_meteor/__init__.py,sha256=209na3pPsdmcuYpYHUYtqQybCpc3yZkc93HnRdicSlI,266
+azure/ai/evaluation/_evaluators/_meteor/_meteor.py,sha256=62AD1DVH_T_t9MDK4GEcvujC0Aka-SEggxEAmc68byg,3315
+azure/ai/evaluation/_evaluators/_protected_material/__init__.py,sha256=eRAQIU9diVXfO5bp6aLWxZoYUvOsrDIfy1gnDOeNTiI,109
+azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py,sha256=6eZXjp2iXSaTNCx_8_8A96lhH5cdm3rmeJfvzMJHV9k,4233
+azure/ai/evaluation/_evaluators/_protected_materials/__init__.py,sha256=A12UsRVIebGvy9FtZLBPsOIAWUskBt8iuhRdILyRcSo,112
+azure/ai/evaluation/_evaluators/_protected_materials/_protected_materials.py,sha256=pRB48BwbnCuozVo7hpblh_ZnApWzmrqiTx0JqhH4ioY,4204
+azure/ai/evaluation/_evaluators/_qa/__init__.py,sha256=bcXfT--C0hjym2haqd1B2-u9bDciyM0ThOFtU1Q69sk,244
+azure/ai/evaluation/_evaluators/_qa/_qa.py,sha256=QSpk4wKIBdwVaI0-u4iRVlnezgxhPnSEAWRUxSwOJM8,3792
+azure/ai/evaluation/_evaluators/_relevance/__init__.py,sha256=JlxytW32Nl8pbE-fI3GRpfgVuY9EG6zxIAn5VZGSwyc,265
+azure/ai/evaluation/_evaluators/_relevance/_relevance.py,sha256=YFLxC8VQ9kvEGK6jge7_yQCxXM1yPChy9d-dANVJA60,4507
+azure/ai/evaluation/_evaluators/_relevance/relevance.prompty,sha256=AO70ho2nMhBtKcl_q4nKFW4kA1LjYsmSfymNa-Cbcrw,3735
+azure/ai/evaluation/_evaluators/_rouge/__init__.py,sha256=kusCDaYcXogDugGefRP8MQSn9xv107oDbrMCqZ6K4GA,291
+azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=IbD10QZMYzaXKqr6q4Diy7p06yhJ3kT8pMv2jnyMNtQ,3444
+azure/ai/evaluation/_evaluators/_similarity/__init__.py,sha256=V2Mspog99_WBltxTkRHG5NpN5s9XoiTSN4I8POWEkLA,268
+azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=z_mUu0lL_gZI041PZvIBgSZ9Aj7j8wUgyy5OPFlBJ_w,4495
+azure/ai/evaluation/_evaluators/_similarity/similarity.prompty,sha256=p2Tb4IW6QnP2BaGRQsAicW4V0B23Oezhf5l3Hau0nxU,4770
+azure/ai/evaluation/_evaluators/_xpia/__init__.py,sha256=VMEL8WrpJQeh4sQiOLzP7hRFPnjzsvwfvTzaGCVJPCM,88
+azure/ai/evaluation/_evaluators/_xpia/xpia.py,sha256=-A9EpdZXr5wZzo_-cF_OX8mMAjy8AgKjNXt1D6Up0NU,5819
+azure/ai/evaluation/simulator/__init__.py,sha256=4O7O6esevfOX0Vb7Eo6nl_m9Qx_WDytwfSgZ7-E1HkE,485
+azure/ai/evaluation/simulator/_adversarial_scenario.py,sha256=SxpyMw5wmM5-fiUjl1_oJH0GQEnsa7ASso10MAr2Hjw,1030
+azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=toUDdMOD4JrsTOh9tsFTLTBz87o6YLgoY6djXnbO7Fk,20577
+azure/ai/evaluation/simulator/_constants.py,sha256=xM-Or2x7RytfoeBM3N7Vt4JQDJX66UdL3CPz0YN5rvE,485
+azure/ai/evaluation/simulator/_direct_attack_simulator.py,sha256=ACd4HZBtY18stKBTGW7iHEq0hn42gNK453V_pioOcxA,11702
+azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=4HX_034m2j8CBNBVSap9CfwKJX00l0fbvuXJyuZYavU,9639
+azure/ai/evaluation/simulator/_tracing.py,sha256=svCnjlI09AQ4aNH8yeXZiE6RKg9Pi4q66oQ8LstKJBk,2968
+azure/ai/evaluation/simulator/_utils.py,sha256=v-_nzpNj8RYnSyBeXcdxqu-jj1Xj_UgYdc21Gty8uwY,4219
+azure/ai/evaluation/simulator/simulator.py,sha256=902a7P5DGrkN1blbCTzHGDEFhA60LlxdjVEJVVa4cSQ,28464
+azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=Qxu2x1cJ2fPTQtYhhNP5C4-d8P33Sru4Zn6xAgroDso,12774
+azure/ai/evaluation/simulator/_conversation/_conversation.py,sha256=9a2plS1QzyuvWbJYj6rY128401e6d_n_yiBNekIDkww,7251
+azure/ai/evaluation/simulator/_conversation/constants.py,sha256=3v7zkjPwJAPbSpJYIK6VOZZy70bJXMo_QTVqSFGlq9A,984
+azure/ai/evaluation/simulator/_helpers/__init__.py,sha256=CD2fGrUgEE3puIm5QYqbkCN1AtntpF-3hRbF98hkhqE,203
+azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py,sha256=7BBLH78b7YDelHDLbAIwf-IO9s9cAEtn-RRXmNReHdc,1017
+azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py,sha256=NjsmHy-bb0yeqKVz9j7f7KIlTEB7MeZe1NzFOLBdBm4,2531
+azure/ai/evaluation/simulator/_model_tools/__init__.py,sha256=aMv5apb7uVjuhMF9ohhA5kQmo652hrGIJlhdl3y2R1I,835
+azure/ai/evaluation/simulator/_model_tools/_identity_manager.py,sha256=qhYmG2r4IAKDePtf9DdqgvNGYlc0xjH4x5sShGxS-lA,5070
+azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py,sha256=fHE6OUc1drkABwGQ5L0ctrqARPjv5uwO-u7yOJhdCuQ,8397
+azure/ai/evaluation/simulator/_model_tools/_rai_client.py,sha256=ITPiciPQQkZa5s2E8pa-x3lKrf8rp7sbr_WSPpECOnc,6725
+azure/ai/evaluation/simulator/_model_tools/_template_handler.py,sha256=hpxBYui1eaw9_lDJJMnKj8CSWFDXiTa7XAKXA4lS0J4,5482
+azure/ai/evaluation/simulator/_model_tools/models.py,sha256=5pUESJR5b5rzp9W7B3Aaokz_HLHTSLjR13YKumLAxqs,21813
+azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=lNSioz2XiGQJb-AXRNYm2JCLMivZKa3JlHfol2Jd7fY,2244
+azure/ai/evaluation/simulator/_prompty/task_simulate.prompty,sha256=00zLVfNgHZdlbC2XvBedSrwDJOaAhl3B1ohE3LKsGg4,928
+azure_ai_evaluation-1.0.0b1.dist-info/METADATA,sha256=1XGkdZ7U-s9NVdHGktn_nk5S8BBUUQA-jrf-_VUpdVU,12268
+azure_ai_evaluation-1.0.0b1.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+azure_ai_evaluation-1.0.0b1.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
+azure_ai_evaluation-1.0.0b1.dist-info/RECORD,,

{azure_ai_evaluation-1.0.0.dist-info → azure_ai_evaluation-1.0.0b1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.3)
+Generator: setuptools (72.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

azure/ai/evaluation/_common/_experimental.py

@@ -1,172 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import os
-import functools
-import inspect
-import logging
-import sys
-from typing import Callable, Type, TypeVar, Union, overload
-
-from typing_extensions import ParamSpec, TypeGuard
-
-DOCSTRING_TEMPLATE = ".. note::    {0} {1}\n\n"
-DOCSTRING_DEFAULT_INDENTATION = 8
-EXPERIMENTAL_CLASS_MESSAGE = "This is an experimental class,"
-EXPERIMENTAL_METHOD_MESSAGE = "This is an experimental method,"
-EXPERIMENTAL_FIELD_MESSAGE = "This is an experimental field,"
-EXPERIMENTAL_LINK_MESSAGE = (
-    "and may change at any time. Please see https://aka.ms/azuremlexperimental for more information."
-)
-
-_warning_cache = set()
-module_logger = logging.getLogger(__name__)
-
-P = ParamSpec("P")
-T = TypeVar("T")
-
-
-@overload
-def experimental(wrapped: Type[T]) -> Type[T]: ...
-
-
-@overload
-def experimental(wrapped: Callable[P, T]) -> Callable[P, T]: ...
-
-
-def experimental(wrapped: Union[Type[T], Callable[P, T]]) -> Union[Type[T], Callable[P, T]]:
-    """Add experimental tag to a class or a method.
-
-    :param wrapped: Either a Class or Function to mark as experimental
-    :type wrapped: Union[Type[T], Callable[P, T]]
-    :return: The wrapped class or method
-    :rtype: Union[Type[T], Callable[P, T]]
-    """
-
-    def is_class(t: Union[Type[T], Callable[P, T]]) -> TypeGuard[Type[T]]:
-        return isinstance(t, type)
-
-    if is_class(wrapped):
-        return _add_class_docstring(wrapped)
-    if inspect.isfunction(wrapped):
-        return _add_method_docstring(wrapped)
-    return wrapped
-
-
-def _add_class_docstring(cls: Type[T]) -> Type[T]:
-    """Add experimental tag to the class doc string.
-
-    :return: The updated class
-    :rtype: Type[T]
-    """
-
-    P2 = ParamSpec("P2")
-
-    def _add_class_warning(func: Callable[P2, None]) -> Callable[P2, None]:
-        """Add warning message for class __init__.
-
-        :param func: The original __init__ function
-        :type func: Callable[P2, None]
-        :return: Updated __init__
-        :rtype: Callable[P2, None]
-        """
-
-        @functools.wraps(func)
-        def wrapped(*args, **kwargs):
-            message = "Class {0}: {1} {2}".format(cls.__name__, EXPERIMENTAL_CLASS_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
-            if not _should_skip_warning() and not _is_warning_cached(message):
-                module_logger.warning(message)
-            return func(*args, **kwargs)
-
-        return wrapped
-
-    doc_string = DOCSTRING_TEMPLATE.format(EXPERIMENTAL_CLASS_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
-    if cls.__doc__:
-        cls.__doc__ = _add_note_to_docstring(cls.__doc__, doc_string)
-    else:
-        cls.__doc__ = doc_string + ">"
-    cls.__init__ = _add_class_warning(cls.__init__)  # type: ignore[method-assign]
-    return cls
-
-
-def _add_method_docstring(func: Callable[P, T]) -> Callable[P, T]:
-    """Add experimental tag to the method doc string.
-
-    :param func: The function to update
-    :type func: Callable[P, T]
-    :return: A wrapped method marked as experimental
-    :rtype: Callable[P,T]
-    """
-    doc_string = DOCSTRING_TEMPLATE.format(EXPERIMENTAL_METHOD_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
-    if func.__doc__:
-        func.__doc__ = _add_note_to_docstring(func.__doc__, doc_string)
-    else:
-        # '>' is required. Otherwise the note section can't be generated
-        func.__doc__ = doc_string + ">"
-
-    @functools.wraps(func)
-    def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
-        message = "Method {0}: {1} {2}".format(func.__name__, EXPERIMENTAL_METHOD_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
-        if not _should_skip_warning() and not _is_warning_cached(message):
-            module_logger.warning(message)
-        return func(*args, **kwargs)
-
-    return wrapped
-
-
-def _add_note_to_docstring(doc_string: str, note: str) -> str:
-    """Adds experimental note to docstring at the top and correctly indents original docstring.
-
-    :param doc_string: The docstring
-    :type doc_string: str
-    :param note: The note to add to the docstring
-    :type note: str
-    :return: Updated docstring
-    :rtype: str
-    """
-    indent = _get_indentation_size(doc_string)
-    doc_string = doc_string.rjust(len(doc_string) + indent)
-    return note + doc_string
-
-
-def _get_indentation_size(doc_string: str) -> int:
-    """Finds the minimum indentation of all non-blank lines after the first line.
-
-    :param doc_string: The docstring
-    :type doc_string: str
-    :return: Minimum number of indentation of the docstring
-    :rtype: int
-    """
-    lines = doc_string.expandtabs().splitlines()
-    indent = sys.maxsize
-    for line in lines[1:]:
-        stripped = line.lstrip()
-        if stripped:
-            indent = min(indent, len(line) - len(stripped))
-    return indent if indent < sys.maxsize else DOCSTRING_DEFAULT_INDENTATION
-
-
-def _should_skip_warning():
-    skip_warning_msg = False
-
-    if os.getenv("AI_EVALS_DISABLE_EXPERIMENTAL_WARNING", "false").lower() == "true":
-        skip_warning_msg = True
-
-    # Cases where we want to suppress the warning:
-    # 1. When converting from REST object to SDK object
-    for frame in inspect.stack():
-        if frame.function == "_from_rest_object":
-            skip_warning_msg = True
-            break
-
-    return skip_warning_msg
-
-
-def _is_warning_cached(warning_msg):
-    # use cache to make sure we only print same warning message once under same session
-    # this prevents duplicated warnings got printed when user does a loop call on a method or a class
-    if warning_msg in _warning_cache:
-        return True
-    _warning_cache.add(warning_msg)
-    return False