azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.0.0b4__py3-none-any.whl

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -1,53 +1,24 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import functools
 import logging
-from typing import Callable
-
-from promptflow._sdk._telemetry import ActivityType, monitor_operation
+from typing import Callable, cast
 
+from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation.simulator import AdversarialScenario
-from azure.identity import DefaultAzureCredential
+from azure.core.credentials import TokenCredential
 
 from ._adversarial_simulator import AdversarialSimulator
+from ._helpers import experimental
 from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
 
 logger = logging.getLogger(__name__)
 
 
-def monitor_adversarial_scenario(func) -> Callable:
-    """Decorator to monitor adversarial scenario.
-
-    :param func: The function to be decorated.
-    :type func: Callable
-    :return: The decorated function.
-    :rtype: Callable
-    """
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        scenario = str(kwargs.get("scenario", None))
-        max_conversation_turns = kwargs.get("max_conversation_turns", None)
-        max_simulation_results = kwargs.get("max_simulation_results", None)
-        decorated_func = monitor_operation(
-            activity_name="xpia.adversarial.simulator.call",
-            activity_type=ActivityType.PUBLICAPI,
-            custom_dimensions={
-                "scenario": scenario,
-                "max_conversation_turns": max_conversation_turns,
-                "max_simulation_results": max_simulation_results,
-            },
-        )(func)
-
-        return decorated_func(*args, **kwargs)
-
-    return wrapper
-
-
+@experimental
 class IndirectAttackSimulator:
     """
     Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
@@ -59,41 +30,29 @@ class IndirectAttackSimulator:
     :type credential: ~azure.core.credentials.TokenCredential
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
+    def __init__(self, *, azure_ai_project: dict, credential):
         """Constructor."""
-        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
-        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
-            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name and project_name"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.MISSING_FIELD,
-                blame=ErrorBlame.USER_ERROR,
-            )
-        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
-            msg = "subscription_id, resource_group_name and project_name keys cannot be None"
+
+        try:
+            self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
+        except EvaluationException as e:
             raise EvaluationException(
-                message=msg,
-                internal_message=msg,
+                message=e.message,
+                internal_message=e.internal_message,
                 target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.MISSING_FIELD,
-                blame=ErrorBlame.USER_ERROR,
-            )
-        if "credential" not in azure_ai_project and not credential:
-            credential = DefaultAzureCredential()
-        elif "credential" in azure_ai_project:
-            credential = azure_ai_project["credential"]
-        self.credential = credential
-        self.azure_ai_project = azure_ai_project
+                category=e.category,
+                blame=e.blame,
+            ) from e
+
+        self.credential = cast(TokenCredential, credential)
         self.token_manager = ManagedIdentityAPITokenManager(
             token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
             logger=logging.getLogger("AdversarialSimulator"),
-            credential=credential,
+            credential=self.credential,
         )
-        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
+        self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
         self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=azure_ai_project, rai_client=self.rai_client
+            azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
         )
 
     def _ensure_service_dependencies(self):
@@ -107,7 +66,6 @@ class IndirectAttackSimulator:
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
@@ -192,7 +150,7 @@ class IndirectAttackSimulator:
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
-        jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
+        jb_sim = AdversarialSimulator(azure_ai_project=cast(dict, self.azure_ai_project), credential=self.credential)
         jb_sim_results = await jb_sim(
             scenario=scenario,
             target=target,

azure/ai/evaluation/simulator/_model_tools/_identity_manager.py

@@ -3,13 +3,15 @@
 # ---------------------------------------------------------
 
 import asyncio
+import inspect
 import logging
 import os
 import time
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Dict, Optional, Union
+from typing import Optional, Union
 
+from azure.core.credentials import TokenCredential, AccessToken
 from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
 
 AZURE_TOKEN_REFRESH_INTERVAL = 600  # seconds
@@ -29,24 +31,24 @@ class APITokenManager(ABC):
     :param auth_header: Authorization header prefix. Defaults to "Bearer"
     :type auth_header: str
     :param credential: Azure credential object
-    :type credential: Optional[Union[azure.identity.DefaultAzureCredential, azure.identity.ManagedIdentityCredential]
+    :type credential: Optional[TokenCredential]
     """
 
     def __init__(
         self,
         logger: logging.Logger,
         auth_header: str = "Bearer",
-        credential: Optional[Union[DefaultAzureCredential, ManagedIdentityCredential]] = None,
+        credential: Optional[TokenCredential] = None,
     ) -> None:
         self.logger = logger
         self.auth_header = auth_header
-        self._lock = None
+        self._lock: Optional[asyncio.Lock] = None
         if credential is not None:
             self.credential = credential
         else:
             self.credential = self.get_aad_credential()
-        self.token = None
-        self.last_refresh_time = None
+        self.token: Optional[str] = None
+        self.last_refresh_time: Optional[float] = None
 
     @property
     def lock(self) -> asyncio.Lock:
@@ -73,20 +75,26 @@ class APITokenManager(ABC):
         identity_client_id = os.environ.get("DEFAULT_IDENTITY_CLIENT_ID", None)
         if identity_client_id is not None:
             self.logger.info(f"Using DEFAULT_IDENTITY_CLIENT_ID: {identity_client_id}")
-            credential = ManagedIdentityCredential(client_id=identity_client_id)
-        else:
-            self.logger.info("Environment variable DEFAULT_IDENTITY_CLIENT_ID is not set, using DefaultAzureCredential")
-            credential = DefaultAzureCredential()
-        return credential
+            return ManagedIdentityCredential(client_id=identity_client_id)
+
+        self.logger.info("Environment variable DEFAULT_IDENTITY_CLIENT_ID is not set, using DefaultAzureCredential")
+        return DefaultAzureCredential()
+
+    @abstractmethod
+    def get_token(self) -> str:
+        """Async method to get the API token. Subclasses should implement this method.
+
+        :return: API token
+        :rtype: str
+        """
 
     @abstractmethod
-    async def get_token(self) -> str:
+    async def get_token_async(self) -> str:
         """Async method to get the API token. Subclasses should implement this method.
 
         :return: API token
         :rtype: str
         """
-        pass  # pylint: disable=unnecessary-pass
 
 
 class ManagedIdentityAPITokenManager(APITokenManager):
@@ -100,12 +108,18 @@ class ManagedIdentityAPITokenManager(APITokenManager):
     :paramtype kwargs: Dict
     """
 
-    def __init__(self, token_scope: TokenScope, logger: logging.Logger, **kwargs: Dict):
-        super().__init__(logger, **kwargs)
+    def __init__(
+        self,
+        token_scope: TokenScope,
+        logger: logging.Logger,
+        *,
+        auth_header: str = "Bearer",
+        credential: Optional[TokenCredential] = None,
+    ):
+        super().__init__(logger, auth_header=auth_header, credential=credential)
         self.token_scope = token_scope
 
-    # Bug 3353724: This get_token is sync method, but it is defined as async method in the base class
-    def get_token(self) -> str:  # pylint: disable=invalid-overridden-method
+    def get_token(self) -> str:
         """Get the API token. If the token is not available or has expired, refresh the token.
 
         :return: API token
@@ -122,6 +136,31 @@ class ManagedIdentityAPITokenManager(APITokenManager):
 
         return self.token
 
+    async def get_token_async(self) -> str:
+        """Get the API token synchronously. If the token is not available or has expired, refresh it.
+
+        :return: API token
+        :rtype: str
+        """
+        if (
+            self.token is None
+            or self.last_refresh_time is None
+            or time.time() - self.last_refresh_time > AZURE_TOKEN_REFRESH_INTERVAL
+        ):
+            self.last_refresh_time = time.time()
+            get_token_method = self.credential.get_token(self.token_scope.value)
+            if inspect.isawaitable(get_token_method):
+                # If it's awaitable, await it
+                token_response: AccessToken = await get_token_method
+            else:
+                # Otherwise, call it synchronously
+                token_response = get_token_method
+
+            self.token = token_response.token
+            self.logger.info("Refreshed Azure endpoint token.")
+
+        return self.token
+
 
 class PlainTokenManager(APITokenManager):
     """Plain API Token Manager
@@ -134,11 +173,18 @@ class PlainTokenManager(APITokenManager):
     :paramtype kwargs: Dict
     """
 
-    def __init__(self, openapi_key: str, logger: logging.Logger, **kwargs: Dict):
-        super().__init__(logger, **kwargs)
-        self.token = openapi_key
+    def __init__(
+        self,
+        openapi_key: str,
+        logger: logging.Logger,
+        *,
+        auth_header: str = "Bearer",
+        credential: Optional[TokenCredential] = None,
+    ) -> None:
+        super().__init__(logger, auth_header=auth_header, credential=credential)
+        self.token: str = openapi_key
 
-    async def get_token(self) -> str:
+    def get_token(self) -> str:
         """Get the API token
 
         :return: API token

azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py

@@ -6,13 +6,14 @@ import copy
 import json
 import time
 import uuid
-from typing import Dict, List
+from typing import Any, Dict, List, Optional, cast
 
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client
 from azure.ai.evaluation._user_agent import USER_AGENT
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
 
+from .._model_tools._template_handler import TemplateParameters
 from .models import OpenAIChatCompletionsModel
 
 
@@ -33,7 +34,15 @@ class SimulationRequestDTO:
     :type template_parameters: Dict
     """
 
-    def __init__(self, url, headers, payload, params, templatekey, template_parameters):
+    def __init__(
+        self,
+        url: str,
+        headers: Dict[str, str],
+        payload: Dict[str, Any],
+        params: Dict[str, str],
+        templatekey: str,
+        template_parameters: Optional[TemplateParameters],
+    ):
         self.url = url
         self.headers = headers
         self.json = json.dumps(payload)
@@ -47,9 +56,12 @@ class SimulationRequestDTO:
         :return: The DTO as a dictionary.
         :rtype: Dict
         """
-        if self.templateParameters is not None:
-            self.templateParameters = {str(k): str(v) for k, v in self.templateParameters.items()}
-        return self.__dict__
+        toReturn = self.__dict__.copy()
+
+        if toReturn["templateParameters"] is not None:
+            toReturn["templateParameters"] = {str(k): str(v) for k, v in toReturn["templateParameters"].items()}
+
+        return toReturn
 
     def to_json(self):
         """Convert the DTO to a JSON string.
@@ -73,12 +85,12 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
     :keyword kwargs: Additional keyword arguments to pass to the parent class.
     """
 
-    def __init__(self, name: str, template_key: str, template_parameters, *args, **kwargs) -> None:
+    def __init__(self, name: str, template_key: str, template_parameters: TemplateParameters, **kwargs) -> None:
         self.tkey = template_key
         self.tparam = template_parameters
-        self.result_url = None
+        self.result_url: Optional[str] = None
 
-        super().__init__(name=name, *args, **kwargs)
+        super().__init__(name=name, **kwargs)
 
     def format_request_data(self, messages: List[Dict], **request_params) -> Dict:  # type: ignore[override]
         """Format the request data to query the model with.
@@ -160,7 +172,6 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         }
         # add all additional headers
         headers.update(self.additional_headers)  # type: ignore[arg-type]
-
         params = {}
         if self.api_version:
             params["api-version"] = self.api_version
@@ -184,8 +195,8 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
                 message=f"Received unexpected HTTP status: {response.status_code} {response.text()}", response=response
             )
 
-        response = response.json()
-        self.result_url = response["location"]
+        response_data = response.json()
+        self.result_url = cast(str, response_data["location"])
 
         retry_policy = AsyncRetryPolicy(  # set up retry configuration
             retry_on_status_codes=[202],  # on which statuses to retry
@@ -202,6 +213,12 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         time.sleep(15)
 
         async with get_async_http_client().with_policies(retry_policy=retry_policy) as exp_retry_client:
+            token = await self.token_manager.get_token_async()
+            proxy_headers = {
+                "Authorization": f"Bearer {token}",
+                "Content-Type": "application/json",
+                "User-Agent": USER_AGENT,
+            }
             response = await exp_retry_client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
                 self.result_url, headers=proxy_headers
             )

azure/ai/evaluation/simulator/_model_tools/_template_handler.py

@@ -2,25 +2,66 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Optional
+from typing import Dict, List, Optional, TypedDict, cast
+
+from typing_extensions import NotRequired
 
 from azure.ai.evaluation._model_configurations import AzureAIProject
 
 from ._rai_client import RAIClient
 
-CONTENT_HARM_TEMPLATES_COLLECTION_KEY = set(
-    [
-        "adv_qa",
-        "adv_conversation",
-        "adv_summarization",
-        "adv_search",
-        "adv_rewrite",
-        "adv_content_gen_ungrounded",
-        "adv_content_gen_grounded",
-        "adv_content_protected_material",
-        "adv_politics",
-    ]
-)
+CONTENT_HARM_TEMPLATES_COLLECTION_KEY = {
+    "adv_qa",
+    "adv_conversation",
+    "adv_summarization",
+    "adv_search",
+    "adv_rewrite",
+    "adv_content_gen_ungrounded",
+    "adv_content_gen_grounded",
+    "adv_content_protected_material",
+    "adv_politics",
+}
+
+
+class TemplateParameters(TypedDict):
+    """Parameters used in Templates
+
+    .. note::
+
+        This type is good enough to type check, but is incorrect. It's meant to represent a dictionary with a known
+        `metadata` key (Dict[str, str]), a known `ch_template_placeholder` key (str), and an unknown number of keys
+        that map to `str` values.
+
+        In typescript, this type would be spelled:
+
+        .. code-block:: typescript
+
+            type AdversarialTemplateParameters = {
+                [key: string]: string
+                ch_template_placeholder: string
+                metadata: {[index: string]: string} # Doesn't typecheck but gets the point across
+            }
+
+        At time of writing, this isn't possible to express with a TypedDict. TypedDicts must be "closed" in that
+        they fully specify all the keys they can contain.
+
+        `PEP 728 – TypedDict with Typed Extra Items <https://peps.python.org/pep-0728/>` is a proposal to support
+        this, but would only be available in Python 3.13 at the earliest.
+    """
+
+    metadata: Dict[str, str]
+    conversation_starter: str
+    ch_template_placeholder: str
+    group_of_people: NotRequired[str]
+    category: NotRequired[str]
+    target_population: NotRequired[str]
+    topic: NotRequired[str]
+
+
+class _CategorizedParameter(TypedDict):
+    parameters: List[TemplateParameters]
+    category: str
+    parameters_key: str
 
 
 class ContentHarmTemplatesUtils:
@@ -85,13 +126,19 @@ class AdversarialTemplate:
     :param template_parameters: The template parameters.
     """
 
-    def __init__(self, template_name, text, context_key, template_parameters=None) -> None:
+    def __init__(
+        self,
+        template_name: str,
+        text: Optional[str],
+        context_key: List,
+        template_parameters: Optional[List[TemplateParameters]] = None,
+    ) -> None:
         self.text = text
         self.context_key = context_key
         self.template_name = template_name
-        self.template_parameters = template_parameters
+        self.template_parameters = template_parameters or []
 
-    def __str__(self):
+    def __str__(self) -> str:
         return "{{ch_template_placeholder}}"
 
 
@@ -106,16 +153,13 @@ class AdversarialTemplateHandler:
     """
 
     def __init__(self, azure_ai_project: AzureAIProject, rai_client: RAIClient) -> None:
-        self.cached_templates_source = {}
-        # self.template_env = JinjaEnvironment(loader=JinjaFileSystemLoader(searchpath=template_dir))
         self.azure_ai_project = azure_ai_project
-        self.categorized_ch_parameters = None
+        self.categorized_ch_parameters: Optional[Dict[str, _CategorizedParameter]] = None
         self.rai_client = rai_client
 
-    async def _get_content_harm_template_collections(self, collection_key):
-
+    async def _get_content_harm_template_collections(self, collection_key: str) -> List[AdversarialTemplate]:
         if self.categorized_ch_parameters is None:
-            categorized_parameters = {}
+            categorized_parameters: Dict[str, _CategorizedParameter] = {}
             util = ContentHarmTemplatesUtils
 
             parameters = await self.rai_client.get_contentharm_parameters()
@@ -123,7 +167,7 @@ class AdversarialTemplateHandler:
             for k in parameters.keys():
                 template_key = util.get_template_key(k)
                 categorized_parameters[template_key] = {
-                    "parameters": parameters[k],
+                    "parameters": cast(List[TemplateParameters], parameters[k]),
                     "category": util.get_template_category(k),
                     "parameters_key": k,
                 }

azure/ai/evaluation/simulator/_model_tools/models.py

@@ -49,10 +49,10 @@ class LLMBase(ABC):
     Base class for all LLM models.
     """
 
-    def __init__(self, endpoint_url: str, name: str = "unknown", additional_headers: Optional[dict] = {}):
+    def __init__(self, endpoint_url: str, name: str = "unknown", additional_headers: Optional[Dict[str, str]] = None):
         self.endpoint_url = endpoint_url
         self.name = name
-        self.additional_headers = additional_headers
+        self.additional_headers = additional_headers or {}
         self.logger = logging.getLogger(repr(self))
 
         # Metric tracking
@@ -208,7 +208,7 @@ class OpenAICompletionsModel(LLMBase):
         *,
         endpoint_url: str,
         name: str = "OpenAICompletionsModel",
-        additional_headers: Optional[dict] = {},
+        additional_headers: Optional[Dict[str, str]] = None,
         api_version: Optional[str] = "2023-03-15-preview",
         token_manager: APITokenManager,
         azureml_model_deployment: Optional[str] = None,
@@ -220,7 +220,7 @@ class OpenAICompletionsModel(LLMBase):
         frequency_penalty: Optional[float] = 0,
         presence_penalty: Optional[float] = 0,
         stop: Optional[Union[List[str], str]] = None,
-        image_captions: Dict[str, str] = {},
+        image_captions: Optional[Dict[str, str]] = None,
         images_dir: Optional[str] = None,  # Note: unused, kept for class compatibility
     ):
         super().__init__(endpoint_url=endpoint_url, name=name, additional_headers=additional_headers)
@@ -234,7 +234,7 @@ class OpenAICompletionsModel(LLMBase):
         self.n = n
         self.frequency_penalty = frequency_penalty
         self.presence_penalty = presence_penalty
-        self.image_captions = image_captions
+        self.image_captions = image_captions or {}
 
         # Default stop to end token if not provided
         if not stop:
@@ -263,7 +263,7 @@ class OpenAICompletionsModel(LLMBase):
     def get_model_params(self):
         return {param: getattr(self, param) for param in self.model_param_names if getattr(self, param) is not None}
 
-    def format_request_data(self, prompt: str, **request_params) -> Dict[str, str]:
+    def format_request_data(self, prompt: Dict[str, str], **request_params) -> Dict[str, str]:  # type: ignore[override]
         """
         Format the request data for the OpenAI API.
         """
@@ -328,7 +328,7 @@ class OpenAICompletionsModel(LLMBase):
         # Format prompts and tag with index
         request_datas: List[Dict] = []
         for idx, prompt in enumerate(prompts):
-            prompt: Dict[str, str] = self.format_request_data(prompt, **request_params)
+            prompt = self.format_request_data(prompt, **request_params)
             prompt[self.prompt_idx_key] = idx  # type: ignore[assignment]
             request_datas.append(prompt)
 
@@ -447,7 +447,7 @@ class OpenAICompletionsModel(LLMBase):
 
         self._log_request(request_data)
 
-        token = await self.token_manager.get_token()
+        token = self.token_manager.get_token()
 
         headers = {
             "Content-Type": "application/json",
@@ -522,8 +522,8 @@ class OpenAIChatCompletionsModel(OpenAICompletionsModel):
     formats the prompt for chat completion.
     """
 
-    def __init__(self, name="OpenAIChatCompletionsModel", *args, **kwargs):
-        super().__init__(name=name, *args, **kwargs)
+    def __init__(self, name="OpenAIChatCompletionsModel", **kwargs):
+        super().__init__(name=name, **kwargs)
 
     def format_request_data(self, messages: List[dict], **request_params):  # type: ignore[override]
         request_data = {"messages": messages, **self.get_model_params()}

azure/ai/evaluation/simulator/_prompty/task_query_response.prompty

@@ -3,11 +3,6 @@ name: TaskSimulatorQueryResponse
 description: Gets queries and responses from a blob of text
 model:
   api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    api_key: ${env:AZURE_OPENAI_API_KEY}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     top_p: 1.0
@@ -33,7 +28,8 @@ Answer must not be more than 5 words
 Answer must be picked from Text as is
 Question should be as descriptive as possible and must include as much context as possible from Text
 Output must always have the provided number of QnAs
-Output must be in JSON format
+Output must be in JSON format.
+Output must have {{num_queries}} objects in the format specified below. Any other count is unacceptable.
 Text:
 <|text_start|>
 On January 24, 1984, former Apple CEO Steve Jobs introduced the first Macintosh. In late 2003, Apple had 2.06 percent of the desktop share in the United States.

azure/ai/evaluation/simulator/_prompty/task_simulate.prompty

@@ -3,10 +3,6 @@ name: TaskSimulatorWithPersona
 description: Simulates a user to complete a conversation
 model:
   api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     top_p: 1.0