fiddler-client 2.4.0.dev2__py3-none-any.whl → 2.5.0.dev1__py3-none-any.whl

fiddler3/entities/xai.py

@@ -9,8 +9,11 @@ from uuid import UUID
 
 import pandas as pd
 
+from fiddler3.constants.dataset import EnvType
 from fiddler3.constants.xai import ExplainMethod
 from fiddler3.decorators import handle_api_error
+from fiddler3.entities.job import Job
+from fiddler3.schemas.job import JobCompactResp
 from fiddler3.schemas.xai import (
     DatasetDataSource,
     EventIdDataSource,
@@ -18,12 +21,19 @@ from fiddler3.schemas.xai import (
     SqlSliceQueryDataSource,
 )
 from fiddler3.utils.helpers import try_series_retype
+from fiddler3.utils.logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class XaiMixin:
     id: UUID | None
     _client: Callable
 
+    def _get_method(self, update: bool = False) -> Callable:
+        """Get HTTP method"""
+        return self._client().put if update else self._client().post
+
     @handle_api_error
     def explain(  # pylint: disable=too-many-arguments
         self,
@@ -273,6 +283,221 @@ class XaiMixin:
         )
         return pd.DataFrame(response.json()['data']['predictions'])
 
+    @handle_api_error
+    def get_feature_impact(  # pylint: disable=too-many-arguments
+        self,
+        data_source: DatasetDataSource | SqlSliceQueryDataSource,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+        min_support: int | None = None,
+        output_columns: list[str] | None = None,
+    ) -> tuple:
+        """
+        Get global feature impact for a model over a dataset or a slice.
+
+        :param data_source: DataSource for the input dataset to compute feature
+            impact on (DatasetDataSource or SqlSliceQueryDataSource)
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+        :param min_support: Only used for NLP (TEXT inputs) models. Specify a minimum
+            support (number of times a specific word was present in the sample data)
+            to retrieve top words. Default to 15.
+        :param output_columns: Only used for NLP (TEXT inputs) models. Output column
+            names to compute feature impact on.
+
+        :return: Feature Impact tuple
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'data_source': data_source.dict(),
+        }
+
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+        if min_support:
+            payload['min_support'] = min_support
+        if output_columns:
+            payload['output_columns'] = output_columns
+
+        response = self._client().post(
+            url='/v3/analytics/feature-impact',
+            data=payload,
+        )
+
+        return namedtuple('FeatureImpact', response.json()['data'])(
+            **response.json()['data']
+        )
+
+    @handle_api_error
+    def get_feature_importance(  # pylint: disable=too-many-arguments
+        self,
+        data_source: DatasetDataSource | SqlSliceQueryDataSource,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+    ) -> tuple:
+        """
+        Get global feature importance for a model over a dataset or a slice.
+
+        :param data_source: DataSource for the input dataset to compute feature
+            importance on (DatasetDataSource or SqlSliceQueryDataSource)
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+
+        :return: Feature Importance tuple
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'data_source': data_source.dict(),
+        }
+
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+
+        response = self._client().post(
+            url='/v3/analytics/feature-importance',
+            data=payload,
+        )
+
+        return namedtuple('FeatureImportance', response.json()['data'])(
+            **response.json()['data']
+        )
+
+    @handle_api_error
+    def precompute_feature_impact(  # pylint: disable=too-many-arguments
+        self,
+        dataset_id: UUID,
+        num_samples: int | None = None,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+        min_support: int | None = None,
+        update: bool = False,
+    ) -> Job:
+        """Pre-compute feature impact for a model on a dataset.
+
+        This is used in various places in the UI.
+        A single feature impact can be precomputed (computed and cached) for a model.
+
+        :param dataset_id: The unique identifier of the dataset
+        :param num_samples: The number of samples used
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+        :param min_support: Only used for NLP (TEXT inputs) models. Specify a minimum
+            support (number of times a specific word was present in the sample data)
+            to retrieve top words. Default to 15.
+        :param update: Whether the precomputed feature impact should be recomputed and updated
+
+        :return: Async Job
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_id': dataset_id,
+            'env_type': EnvType.PRE_PRODUCTION,
+        }
+        if num_samples:
+            payload['num_samples'] = num_samples
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+        if min_support:
+            payload['min_support'] = min_support
+
+        method = self._get_method(update)
+
+        response = method(
+            url='/v3/analytics/precompute-feature-impact',
+            params=payload,
+        )
+
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        logger.info(
+            'Model[%s] - Submitted job (%s) for precomputing feature impact',
+            self.id,
+            job_compact.id,
+        )
+        return Job.get(id_=job_compact.id)
+
+    @handle_api_error
+    def precompute_feature_importance(  # pylint: disable=too-many-arguments
+        self,
+        dataset_id: UUID,
+        num_samples: int | None = None,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+        update: bool = False,
+    ) -> Job:
+        """Pre-compute feature importance for a model on a dataset.
+
+        This is used in various places in the UI.
+        A single feature importance can be precomputed (computed and cached) for a model.
+
+        :param dataset_id: The unique identifier of the dataset
+        :param num_samples: The number of samples used
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+        :param update: Whether the precomputed feature impact should be recomputed and updated
+
+        :return: Async Job
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_id': dataset_id,
+            'env_type': EnvType.PRE_PRODUCTION,
+        }
+        if num_samples:
+            payload['num_samples'] = num_samples
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+
+        method = self._get_method(update)
+
+        response = method(
+            url='/v3/analytics/precompute-feature-importance',
+            params=payload,
+        )
+
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        logger.info(
+            'Model[%s] - Submitted job (%s) for precomputing feature importance',
+            self.id,
+            job_compact.id,
+        )
+        return Job.get(id_=job_compact.id)
+
     @handle_api_error
     def get_precomputed_feature_importance(self) -> tuple:
         """Get precomputed feature importance for a model"""
@@ -301,6 +526,48 @@ class XaiMixin:
             **response.json()['data']
         )
 
+    @handle_api_error
+    def precompute_predictions(
+        self,
+        dataset_id: UUID,
+        chunk_size: int | None = None,
+        update: bool = False,
+    ) -> Job:
+        """
+        Pre-compute predictions for a model on a dataset
+
+        :param dataset_id: The unique identifier of the dataset
+        :param chunk_size: Chunk size for fetching predictions
+        :param update: Whether the pre-computed predictions should be re-computed and updated for this dataset
+
+        :return: Dataframe of the predictions
+        """
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_id': dataset_id,
+        }
+
+        if chunk_size:
+            payload['batch_size'] = chunk_size
+
+        method = self._get_method(update)
+
+        response = method(
+            url='/v3/analytics/precompute-predictions',
+            params=payload,
+        )
+
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        logger.info(
+            'Model[%s] - Submitted job (%s) for precomputing predictions on dataset[%s]',
+            self.id,
+            job_compact.id,
+            dataset_id,
+        )
+        return Job.get(id_=job_compact.id)
+
     def _check_id_attributes(self) -> None:
         if not self.id:
             raise AttributeError(

fiddler3/libs/json_encoder.py

@@ -8,5 +8,5 @@ class RequestClientJSONEncoder(JSONEncoder):
     def default(self, o: Any) -> Any:
         """Override JSONEncoder.default to support uuid serialization"""
         if isinstance(o, UUID):
-            return o.hex
+            return str(o)
         return super().default(o)

fiddler3/schemas/alert_record.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from typing import Optional
+from uuid import UUID
+
+from pydantic import Field
+
+from fiddler3.schemas.base import BaseModel
+
+
+class AlertRecordResp(BaseModel):
+    id: UUID = Field(alias='uuid')
+    alert_rule_id: UUID = Field(alias='alert_config_uuid')
+    alert_run_start_time: int
+    alert_time_bucket: int
+    alert_value: float
+    baseline_time_bucket: Optional[int]
+    baseline_value: Optional[float]
+    is_alert: bool
+    severity: str
+    failure_reason: str
+    message: str
+    feature_name: Optional[str]
+    alert_record_main_version: int
+    alert_record_sub_version: int
+
+    created_at: datetime
+    updated_at: datetime

fiddler3/schemas/alert_rule.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+from typing import List, Optional, Union
+from uuid import UUID
+
+from pydantic import Field
+
+from fiddler3.constants.alert_rule import AlertCondition, BinSize, CompareTo, Priority
+from fiddler3.schemas.base import BaseModel
+from fiddler3.schemas.baseline import BaselineCompactResp
+from fiddler3.schemas.model import ModelCompactResp
+from fiddler3.schemas.project import ProjectCompactResp
+
+
+class AlertRuleResp(BaseModel):
+    id: UUID = Field(alias='uuid')
+    name: str
+    model: ModelCompactResp
+    project: ProjectCompactResp
+    baseline: Optional[BaselineCompactResp]
+    priority: Union[str, Priority]
+    compare_to: Union[str, CompareTo]
+    metric_id: Union[str, UUID] = Field(alias='metric')
+    critical_threshold: float
+    condition: Union[str, AlertCondition]
+    bin_size: Union[str, BinSize]
+    columns: Optional[List[str]]
+    baseline_id: Optional[UUID]
+    compare_bin_delta: Optional[int]
+    warning_threshold: Optional[float]
+
+    created_at: datetime
+    updated_at: datetime = Field(alias='last_updated')

fiddler3/schemas/custom_features.py

@@ -46,6 +46,8 @@ class CustomFeature(BaseModel):
             return TextEmbedding.parse_obj(deserialized_json)
         if feature_type == CustomFeatureType.FROM_IMAGE_EMBEDDING:
             return ImageEmbedding.parse_obj(deserialized_json)
+        if feature_type == CustomFeatureType.ENRICHMENT:
+            return Enrichment.parse_obj(deserialized_json)
 
         raise ValueError(f'Unsupported feature type: {feature_type}')
 
@@ -63,6 +65,10 @@ class CustomFeature(BaseModel):
                 return_dict['source_column'] = self.source_column
                 if isinstance(self, TextEmbedding):
                     return_dict['n_tags'] = self.n_tags
+        elif isinstance(self, Enrichment):
+            return_dict['columns'] = self.columns
+            return_dict['enrichment'] = self.enrichment
+            return_dict['config'] = self.config
         else:
             raise ValueError(f'Unsupported feature type: {self.type} {type(self)}')
 
@@ -88,3 +94,31 @@ class TextEmbedding(VectorFeature):
 
 class ImageEmbedding(VectorFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_IMAGE_EMBEDDING
+
+
+class Enrichment(CustomFeature):
+    """
+    Represents an enrichment feature in a data processing or machine learning context.
+
+    This class inherits from CustomFeature and is used to apply specific enrichment
+    operations to a set of columns in a dataset. The type of enrichment and its
+    configuration can be specified.
+
+    Attributes:
+        type (CustomFeatureType): The type of the custom feature, set to ENRICHMENT.
+        columns (List[str]): The list of input column names in the dataset used to generate the enrichment.
+        enrichment (str): A string identifier for the type of enrichment to be applied.
+        config (Dict[str, Any]): A dictionary containing configuration options for the enrichment.
+    """
+
+    # Setting the feature type to ENRICHMENT
+    type: CustomFeatureType = CustomFeatureType.ENRICHMENT
+
+    # List of input column names used to generate the enrichment
+    columns: List[str]
+
+    # String identifier for the enrichment to be applied. e.g. "embedding" or "toxicity"
+    enrichment: str
+
+    # Dictionary for additional configuration options for the enrichment
+    config: Dict[str, Any] = {}

fiddler3/schemas/events.py

@@ -0,0 +1,15 @@
+from typing import Any, Dict, List
+from uuid import UUID
+
+from fiddler3.constants.events import PublishEventsSourceType
+from fiddler3.schemas.base import BaseModel
+
+
+class FileSource(BaseModel):
+    file_id: UUID
+    type: PublishEventsSourceType = PublishEventsSourceType.FILE
+
+
+class EventsSource(BaseModel):
+    events: List[Dict[Any, Any]]
+    type: PublishEventsSourceType = PublishEventsSourceType.EVENTS

fiddler3/schemas/files.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import Field
+
+from fiddler3.schemas.base import BaseModel
+from fiddler3.schemas.user import UserCompactResp
+
+
+class FileResp(BaseModel):
+    id: UUID
+    name: str = Field(alias='filename')
+    status: str
+    type: str
+    created_at: datetime
+    updated_at: datetime
+    created_by: UserCompactResp
+    updated_by: UserCompactResp
+
+
+class FileCompactResp(BaseModel):
+    id: UUID
+    name: str

fiddler3/schemas/model.py

@@ -5,7 +5,7 @@ from uuid import UUID
 from pydantic.fields import Field
 
 from fiddler3.schemas.base import BaseModel
-from fiddler3.schemas.model_schema import Column, ModelSchema
+from fiddler3.schemas.model_schema import ModelSchema
 from fiddler3.schemas.model_spec import ModelSpec
 from fiddler3.schemas.model_task_params import ModelTaskParams
 from fiddler3.schemas.organization import OrganizationCompactResp
@@ -40,9 +40,4 @@ class ModelResp(BaseModel):
     updated_by: UserCompactResp
     organization: OrganizationCompactResp
     project: ProjectCompactResp
-    input_cols: List[Column]
-    output_cols: List[Column]
-    target_cols: List[Column]
-    metadata_cols: List[Column]
-    decision_cols: List[Column]
     is_binary_ranking_model: bool

fiddler3/schemas/webhook.py

@@ -0,0 +1,26 @@
+import enum
+from datetime import datetime
+from uuid import UUID
+
+from pydantic.fields import Field
+
+from fiddler3.schemas.base import BaseModel
+from fiddler3.schemas.user import UserCompactResp
+
+
+@enum.unique
+class WebhookProvider(str, enum.Enum):
+    # provider is 'SLACK' or 'OTHER' as of Aug 2023.
+    SLACK = 'SLACK'
+    OTHER = 'OTHER'
+
+
+class WebhookResp(BaseModel):
+    id: UUID = Field(alias='uuid')
+    name: str
+    url: str
+    provider: WebhookProvider
+    created_at: datetime
+    updated_at: datetime
+    created_by: UserCompactResp
+    updated_by: UserCompactResp

fiddler3/tests/apis/test_alert_record.py

@@ -0,0 +1,99 @@
+from datetime import datetime
+
+import responses
+
+from fiddler3.entities.alert_record import AlertRecord
+from fiddler3.tests.constants import ALERT_RULE_ID, URL
+
+ALERT_RECORD_LIST_RESPONSE = {
+    'data': {
+        'page_size': 100,
+        'total': 2,
+        'item_count': 2,
+        'page_count': 1,
+        'page_index': 1,
+        'offset': 0,
+        'items': [
+            {
+                'id': 26410,
+                'uuid': 'b520da6b-c01c-4375-bde6-2560cd972485',
+                'alert_config_uuid': ALERT_RULE_ID,
+                'alert_run_start_time': 1703255503133,
+                'alert_time_bucket': 1703116800000,
+                'baseline_time_bucket': None,
+                'baseline_value': None,
+                'is_alert': True,
+                'severity': 'CRITICAL',
+                'failure_reason': 'NA',
+                'message': '',
+                'alert_value': 37.0,
+                'feature_name': 'estimatedsalary',
+                'alert_record_main_version': 1,
+                'alert_record_sub_version': 2,
+                'created_at': '2023-12-22T14:31:43.499697+00:00',
+                'updated_at': '2023-12-22T14:31:43.499697+00:00',
+            },
+            {
+                'id': 26418,
+                'uuid': '6e55540c-9053-4af8-b20e-1728b1436f35',
+                'alert_config_uuid': ALERT_RULE_ID,
+                'alert_run_start_time': 1703259102527,
+                'alert_time_bucket': 1703116800000,
+                'baseline_time_bucket': None,
+                'baseline_value': None,
+                'is_alert': True,
+                'severity': 'CRITICAL',
+                'failure_reason': 'NA',
+                'message': '',
+                'alert_value': 37.0,
+                'feature_name': 'estimatedsalary',
+                'alert_record_main_version': 1,
+                'alert_record_sub_version': 3,
+                'created_at': '2023-12-22T15:31:42.698623+00:00',
+                'updated_at': '2023-12-22T15:31:42.698623+00:00',
+            },
+        ],
+    },
+    'api_version': '2.0',
+    'kind': 'PAGINATED',
+}
+
+ALERT_RECORD_EMPTY_LIST_RESPONSE = {
+    'data': {
+        'page_size': 100,
+        'total': 0,
+        'item_count': 0,
+        'page_count': 1,
+        'page_index': 1,
+        'offset': 0,
+        'items': [],
+    },
+    'api_version': '2.0',
+    'kind': 'PAGINATED',
+}
+
+
+@responses.activate
+def test_alert_record_list_success() -> None:
+    responses.get(
+        url=f'{URL}/v2/alert-configs/{ALERT_RULE_ID}/records',
+        json=ALERT_RECORD_LIST_RESPONSE,
+    )
+    alert_records = AlertRecord.list(
+        alert_rule_id=ALERT_RULE_ID,
+        start_time=datetime(2023, 12, 18),
+        end_time=datetime(2023, 12, 25),
+    )
+    for record in alert_records:
+        assert isinstance(record, AlertRecord)
+
+
+@responses.activate
+def test_alert_record_list_success() -> None:
+    responses.get(
+        url=f'{URL}/v2/alert-configs/{ALERT_RULE_ID}/records',
+        json=ALERT_RECORD_EMPTY_LIST_RESPONSE,
+    )
+    alert_records = list(AlertRecord.list(alert_rule_id=ALERT_RULE_ID))
+
+    assert len(alert_records) == 0