fiddler-client 2.4.0.dev2__tar.gz → 2.5.0.dev2__tar.gz

{fiddler-client-2.4.0.dev2/fiddler_client.egg-info → fiddler-client-2.5.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fiddler-client
-Version: 2.4.0.dev2
+Version: 2.5.0.dev2
 Summary: Python client for Fiddler Service
 Home-page: https://fiddler.ai
 Author: Fiddler Labs
@@ -42,9 +42,17 @@ Documentation for the API can be found [here](https://docs.fiddler.ai/reference/
 
 Version History
 -------------
+### 2.5.0
+  - #### **New Features**
+    - Add support for enrichments
+    - Allow pausing multiple alerts at once
+    - Add percentage violation metrics for alert
+
 ### 2.4.0
- - #### **New Features**
-  - Support for enrichments
+  - ### **New Features**
+    - Add support for segments
+  - #### **Modifications**
+    - Ensure `TextEmbedding` and `ImageEmbedding` is supported for vectors in metadata
 
 ### 2.3.0
   - #### **New Features**
@@ -57,8 +65,10 @@ Version History
     - Relax pydantic version to allow any version between 1.9 and 2
 
 ### 2.2.0
- - #### **New Features**
-  - Support segments
+  - #### **Modifications**
+    - `add_custom_metric()` supports an optional `description` parameter
+    - `fql` is renamed to `definition` in `add_custom_metric()`
+    - A new `get_custom_metric()` function to get details about a single custom metric.
 
 ### 2.1.2
   - #### **Modifications**

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/PUBLIC.md

@@ -19,9 +19,17 @@ Documentation for the API can be found [here](https://docs.fiddler.ai/reference/
 
 Version History
 -------------
+### 2.5.0
+  - #### **New Features**
+    - Add support for enrichments
+    - Allow pausing multiple alerts at once
+    - Add percentage violation metrics for alert
+
 ### 2.4.0
- - #### **New Features**
-  - Support for enrichments
+  - ### **New Features**
+    - Add support for segments
+  - #### **Modifications**
+    - Ensure `TextEmbedding` and `ImageEmbedding` is supported for vectors in metadata
 
 ### 2.3.0
   - #### **New Features**
@@ -34,8 +42,10 @@ Version History
     - Relax pydantic version to allow any version between 1.9 and 2
 
 ### 2.2.0
- - #### **New Features**
-  - Support segments
+  - #### **Modifications**
+    - `add_custom_metric()` supports an optional `description` parameter
+    - `fql` is renamed to `definition` in `add_custom_metric()`
+    - A new `get_custom_metric()` function to get details about a single custom metric.
 
 ### 2.1.2
   - #### **Modifications**

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler/__init__.py

@@ -50,6 +50,7 @@ from fiddler.schemas.custom_features import (
     Multivariate,
     TextEmbedding,
     VectorFeature,
+    Enrichment,
 )
 from fiddler.utils import ColorLogger
 from fiddler.utils.logger import get_logger
@@ -71,6 +72,7 @@ __all__ = [
     'VectorFeature',
     'TextEmbedding',
     'ImageEmbedding',
+    'Enrichment',
     'ColorLogger',
     'DatasetInfo',
     'DataType',

fiddler-client-2.5.0.dev2/fiddler/_version.py

@@ -0,0 +1 @@
+__version__ = '2.5.0.dev2'

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler/api/alert_mixin.py

@@ -6,6 +6,8 @@ from typing import Any, Dict, List, Optional, Union
 
 from pydantic import parse_obj_as
 
+import fiddler3
+from fiddler3.entities.model import Model
 from fiddler.libs.http_client import RequestClient
 from fiddler.schema.alert import (
     AlertCondition,
@@ -38,6 +40,8 @@ class AlertMixin:
     client: RequestClient
     organization_name: str
     server_info: ServerInfo
+    url: str
+    auth_token: str
 
     @handle_api_error_response
     def add_alert_rule(
@@ -61,7 +65,7 @@ class AlertMixin:
         warning_threshold: Optional[float] = None,
         notifications_config: Optional[Dict[str, Any]] = None,
     ) -> AlertRule:
-        """
+        f"""
         To add an alert rule
 
         :param project_id: Unique project name for which the alert rule is created
@@ -488,3 +492,49 @@ class AlertMixin:
             },
             'webhooks': webhooks_dict,
         }
+
+    def update_alert_notification_status(
+        self,
+        notification_status: bool,
+        alert_config_ids: Optional[list[str]] = None,
+        model_id: Optional[str] = None,
+    ) -> list[AlertRuleWithNotifications]:
+        """
+        To enable/disable the notifications for a list of Alert Ids or for a given Model Id.
+        Either the Alert Ids should be given or the Model Id, but not both.
+        :param notification_status: The value of notification status for an alert
+        :param alert_config_ids: List of Alert Ids, on which we need to update notification
+        :param model_id: The Model Id which for whom we want to update all alerts.
+        :return: List of updated Alert Rules.
+        """
+        if not alert_config_ids and not model_id:
+            raise ValueError('Either alert_config_ids or model_id must be specified')
+        elif alert_config_ids and model_id:
+            raise ValueError(
+                'Either alert_config_ids or model_id should be specified, not both.'
+            )
+
+        updated_alert_configs: list[AlertRuleWithNotifications] = []
+        alert_ids: list[str] = []
+
+        if alert_config_ids:
+            # verify that all the alert-configs are valid
+            for alert_id in alert_config_ids:
+                self.client.get(
+                    url=f'alert-configs/{alert_id}',
+                )
+            alert_ids = alert_config_ids
+
+        if model_id:
+            fiddler3.init(url=self.url, token=self.auth_token)
+            model: Model = Model.get(id_=model_id)
+            alerts: list[AlertRule] = self.get_alert_rules(model_id=model.name)
+            alert_ids = [alert.alert_rule_uuid for alert in alerts]
+
+        for alert_id in alert_ids:
+            updated_alert_configs.append(
+                self.update_alert_rule(
+                    alert_config_uuid=alert_id, enable_notification=notification_status
+                )
+            )
+        return updated_alert_configs

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler/api/webhooks_mixin.py

@@ -1,5 +1,5 @@
 from http import HTTPStatus
-from typing import List
+from typing import Dict, List
 
 from pydantic import parse_obj_as
 
@@ -80,13 +80,11 @@ class WebhookMixin:
 
         :returns: Updated `Webhook` object.
         """
-        request_body = Webhook(
-            uuid=uuid,
-            organization_name=self.organization_name,
-            name=name,
-            url=url,
-            provider=provider,
-        ).dict()
+        request_body: Dict[str, str] = {
+            'name': name,
+            'url': url,
+            'provider': provider,
+        }
         response = self.client.patch(
             url=f'webhooks/{uuid}',
             data=request_body,

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler/core_objects.py

@@ -28,6 +28,7 @@ from pandas.api.types import CategoricalDtype
 
 from fiddler.schemas.custom_features import (
     CustomFeature,
+    Enrichment,
     ImageEmbedding,
     Multivariate,
     TextEmbedding,
@@ -1231,10 +1232,10 @@ class ModelInfo:
                     'The custom_features argument only accepts a list of CustomFeature objects.'
                 )
 
-            available_cols = (
-                self.get_input_names()
-                + self.get_target_names()
-                + self.get_metadata_names()
+            available_cols: Set[str] = set().union(
+                self.get_input_names(),
+                self.get_target_names(),
+                self.get_metadata_names(),
             )
 
             numeric_cols = self.get_input_pandas_dtypes()
@@ -1243,19 +1244,18 @@ class ModelInfo:
             if self.targets:
                 numeric_cols.update(self.get_target_pandas_dtypes())
 
-            valid_numeric_names = [
+            valid_numeric_names = set(
                 k
                 for k, v in numeric_cols.items()
                 if v in (DataType.INTEGER.value, DataType.FLOAT.value)
-            ]
-
-            valid_vector_cols = [
+            )
+            valid_vector_cols = set(
                 vec_col.name
                 for vec_col in (self.inputs or []) + (self.metadata or [])
                 if vec_col.data_type.is_vector()
-            ]
+            )
 
-            unique_cf_names = []
+            valid_cf_names, valid_enrichment_names = set(), set()
             for feature in self.custom_features:
                 if not isinstance(feature, CustomFeature):
                     raise ValueError(
@@ -1267,42 +1267,92 @@ class ModelInfo:
                         f'A column name "{feature.name}" already exists in the dataset. Please use a different name for custom feature {feature}.'
                     )
 
-                if feature.name in unique_cf_names:
+                if feature.name in valid_cf_names:
                     raise ValueError(
                         f'Multiple custom features are defined with the same name {feature.name}.'
                     )
-                else:
-                    unique_cf_names.append(feature.name)
+                if isinstance(feature, Enrichment):
+                    valid_enrichment_names.add(feature.name)
+                valid_cf_names.add(feature.name)
 
+            for feature in self.custom_features:
                 if isinstance(feature, Multivariate):
                     self.validate_multivariate_cf(feature, valid_numeric_names)
                 elif isinstance(feature, TextEmbedding):
-                    self.validate_text_embedding_cf(
+                    self.validate_embedding_cf(
                         feature,
                         available_cols,
                         valid_vector_cols,
+                        valid_enrichment_names,
+                    )
+                elif isinstance(feature, ImageEmbedding):
+                    self.validate_embedding_cf(
+                        feature,
+                        available_cols,
+                        valid_vector_cols,
+                        [],
                     )
                 elif isinstance(feature, VectorFeature):
                     self.validate_vector_cf(feature, valid_vector_cols)
+                elif isinstance(feature, Enrichment):
+                    self.validate_enrichment_cf(feature, available_cols, valid_cf_names)
 
-    def validate_text_embedding_cf(
+    def validate_enrichment_cf(
         self,
-        feature: TextEmbedding,
-        available_cols: List[str],
-        valid_vector_cols: List[str],
+        feature: Enrichment,
+        available_cols: Iterable[str],
+        valid_cf_names: Iterable[str],
     ) -> None:
-        if feature.column not in valid_vector_cols:
+        if not feature.columns:
             raise ValueError(
-                f"Column '{feature.column}' defined in {feature.name} ({feature.__class__.__name__}) does not exist or not of type vector."
+                f'{feature.name} ({feature.__class__.__name__}) must specify at least one enrichment/column/custom feature in columns.'
             )
+        for column in feature.columns:
+            if column == feature.name:
+                raise ValueError(
+                    f'{feature.name} ({feature.__class__.__name__}) cannot depend on itself.'
+                )
+            if column not in valid_cf_names and column not in available_cols:
+                raise ValueError(
+                    f"Column '{column}' defined in {feature.name} ({feature.__class__.__name__}) does not exist in the data, as a custom feature or as an enrichment."
+                )
 
-        if feature.source_column not in available_cols:
-            raise ValueError(
-                f'{feature.name} ({feature.__class__.__name__}) needs to specify source_col.'
-            )
+    def validate_embedding_cf(
+        self,
+        feature: Union[TextEmbedding, ImageEmbedding],
+        available_cols: Iterable[str],
+        valid_vector_cols: Iterable[str],
+        valid_enrichment_names: Iterable[str],
+    ) -> None:
+        if isinstance(feature, TextEmbedding):
+            if not feature.n_tags:
+                raise ValueError(
+                    f'{feature.name} ({feature.__class__.__name__}) must specify a number of tags.'
+                )
+
+            if feature.n_tags < 0:
+                raise ValueError(
+                    f'{feature.name} ({feature.__class__.__name__}) must specify a non-negative number of tags.'
+                )
+
+        if feature.column not in valid_vector_cols:
+            # we require either a column or source_column
+            if not feature.source_column:
+                raise ValueError(
+                    f'{feature.name} ({feature.__class__.__name__}) needs to specify source_column.'
+                )
+            if feature.source_column not in available_cols:
+                raise ValueError(
+                    f'source_column {feature.source_column} defined in {feature.name} ({feature.__class__.__name__}) does not exist or cannot be used as a source column.'
+                )
+            # column could be the result of an enrichment
+            if feature.column not in valid_enrichment_names:
+                raise ValueError(
+                    f"Column/Enrichment'{feature.column}' defined in {feature.name} ({feature.__class__.__name__}) does not exist or not of type vector."
+                )
 
     def validate_vector_cf(
-        self, feature: VectorFeature, valid_vector_cols: List[str]
+        self, feature: VectorFeature, valid_vector_cols: Iterable[str]
     ) -> None:
         if feature.column not in valid_vector_cols:
             raise ValueError(
@@ -1310,7 +1360,7 @@ class ModelInfo:
             )
 
     def validate_multivariate_cf(
-        self, feature: Multivariate, valid_numeric_names: List[str]
+        self, feature: Multivariate, valid_numeric_names: Iterable[str]
     ) -> None:
         if len(feature.columns or []) < 2:
             raise ValueError(

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler/schema/alert.py

@@ -34,6 +34,12 @@ class Metric(str, enum.Enum):
     MISSING_VALUE = 'null_violation_count'
     RANGE_VIOLATION = 'range_violation_count'
     TYPE_VIOLATION = 'type_violation_count'
+    ANY_VIOLATION = 'any_violation_count'
+
+    MISSING_VALUE_PCT = 'null_violation_percentage'
+    RANGE_VIOLATION_PCT = 'range_violation_percentage'
+    TYPE_VIOLATION_PCT = 'type_violation_percentage'
+    ANY_VIOLATION_PCT = 'any_violation_percentage'
 
     ## Performance metrics
     # Binary Classification
@@ -47,7 +53,6 @@ class Metric(str, enum.Enum):
     CALIBRATED_THRESHOLD = 'calibrated_threshold'
     GMEAN = 'geometric_mean'
     LOG_LOSS = 'log_loss'
-    
 
     # Regression
     R2 = 'r2'
@@ -192,7 +197,7 @@ class AlertRule(BaseDataSchema):
     time_bucket: Optional[int]
     alert_type_display_name: str
     metric_display_name: str
-    enable_notification: bool
+    enable_notification: Optional[bool]
 
     @root_validator(pre=True)
     def set_feature_names(cls, values: dict) -> dict:
@@ -224,4 +229,6 @@ class TriggeredAlerts(BaseDataSchema):
     message: str
     multi_col_values: Optional[Dict[str, float]]
     feature_name: Optional[str]
+    alert_record_main_version: Optional[int]
+    alert_record_sub_version: Optional[int]
     segment: Optional[Segment]

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler/schemas/custom_features.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional, Type, TypeVar
 
-from pydantic import BaseModel, Extra
+from pydantic import BaseModel, Extra, validator
 
 CustomFeatureTypeVar = TypeVar('CustomFeatureTypeVar', bound='CustomFeature')
 DEFAULT_NUM_CLUSTERS = 5
@@ -22,6 +22,12 @@ class CustomFeature(BaseModel):
     n_clusters: Optional[int] = DEFAULT_NUM_CLUSTERS
     centroids: Optional[List] = None
 
+    @validator('n_clusters')
+    def validate_n_clusters(cls, value: int) -> int:  # noqa: N805
+        if value < 0:
+            raise ValueError('n_clusters must be greater than 0')
+        return value
+
     class Config:
         allow_mutation = False
         extra = Extra.forbid
@@ -47,6 +53,8 @@ class CustomFeature(BaseModel):
             return TextEmbedding.parse_obj(deserialized_json)
         elif feature_type == CustomFeatureType.FROM_IMAGE_EMBEDDING:
             return ImageEmbedding.parse_obj(deserialized_json)
+        elif feature_type == CustomFeatureType.ENRICHMENT:
+            return Enrichment.parse_obj(deserialized_json)
         else:
             raise ValueError(f'Unsupported feature type: {feature_type}')
 
@@ -64,6 +72,10 @@ class CustomFeature(BaseModel):
                 return_dict['source_column'] = self.source_column
                 if isinstance(self, TextEmbedding):
                     return_dict['n_tags'] = self.n_tags
+        elif isinstance(self, Enrichment):
+            return_dict['columns'] = self.columns
+            return_dict['enrichment'] = self.enrichment
+            return_dict['config'] = self.config
         else:
             raise ValueError(f'Unsupported feature type: {self.type} {type(self)}')
 
@@ -75,6 +87,12 @@ class Multivariate(CustomFeature):
     columns: List[str]
     monitor_components: bool = False
 
+    @validator('columns')
+    def validate_columns(cls, value: List[str]) -> List[str]:  # noqa: N805
+        if len(value) < 2:
+            raise ValueError('Multivariate features must have at least 2 columns')
+        return value
+
 
 class VectorFeature(CustomFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_VECTOR
@@ -86,6 +104,31 @@ class TextEmbedding(VectorFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_TEXT_EMBEDDING
     n_tags: Optional[int] = DEFAULT_NUM_TAGS
 
+    @validator('source_column')
+    def validate_source_column(cls, value: str) -> str:  # noqa: N805
+        if value is None:
+            raise ValueError('source_column must be specified')
+        return value
+
+    @validator('n_tags')
+    def validate_n_tags(cls, value: int) -> int:  # noqa: N805
+        if value < 1:
+            raise ValueError('n_tags must be greater than 0')
+        return value
+
 
 class ImageEmbedding(VectorFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_IMAGE_EMBEDDING
+
+    @validator('source_column')
+    def validate_source_column(cls, value: str) -> str:  # noqa: N805
+        if value is None:
+            raise ValueError('source_column must be specified')
+        return value
+
+
+class Enrichment(CustomFeature):
+    type: CustomFeatureType = CustomFeatureType.ENRICHMENT
+    columns: List[str]
+    enrichment: str
+    config: Dict[str, Any] = {}

fiddler-client-2.5.0.dev2/fiddler3/__init__.py

@@ -0,0 +1,122 @@
+from fiddler3.connection import Connection, ConnectionMixin, init  # noqa
+from fiddler3.constants.alert_rule import (  # noqa
+    AlertCondition,
+    BinSize,
+    CompareTo,
+    Priority,
+)
+from fiddler3.constants.baseline import BaselineType, WindowSize  # noqa
+from fiddler3.constants.dataset import EnvType  # noqa
+from fiddler3.constants.job import JobStatus  # noqa
+from fiddler3.constants.model import (  # noqa
+    ArtifactStatus,
+    CustomFeatureType,
+    DataType,
+    ModelInputType,
+    ModelTask,
+)
+from fiddler3.constants.model_deployment import ArtifactType, DeploymentType  # noqa
+from fiddler3.constants.xai import ExplainMethod  # noqa
+from fiddler3.entities.alert_record import AlertRecord  # noqa
+from fiddler3.entities.alert_rule import AlertRule  # noqa
+from fiddler3.entities.baseline import Baseline  # noqa
+from fiddler3.entities.dataset import Dataset  # noqa
+from fiddler3.entities.file import File  # noqa
+from fiddler3.entities.job import Job  # noqa
+from fiddler3.entities.model import Model  # noqa
+from fiddler3.entities.model_deployment import ModelDeployment  # noqa
+from fiddler3.entities.project import Project  # noqa
+from fiddler3.entities.webhook import Webhook  # noqa
+from fiddler3.exceptions import (  # noqa
+    ApiError,
+    AsyncJobFailed,
+    Conflict,
+    ConnectionError,
+    ConnectionTimeout,
+    HttpError,
+    IncompatibleClient,
+    NotFound,
+    Unsupported,
+)
+from fiddler3.schemas.custom_features import (  # noqa
+    CustomFeature,
+    Enrichment,
+    ImageEmbedding,
+    Multivariate,
+    TextEmbedding,
+    VectorFeature,
+)
+from fiddler3.schemas.dataset import EnvType  # noqa
+from fiddler3.schemas.deployment_params import DeploymentParams  # noqa
+from fiddler3.schemas.model_schema import ModelSchema  # noqa
+from fiddler3.schemas.model_spec import ModelSpec  # noqa
+from fiddler3.schemas.model_task_params import ModelTaskParams  # noqa
+from fiddler3.schemas.xai import (  # noqa
+    DatasetDataSource,
+    EventIdDataSource,
+    RowDataSource,
+    SqlSliceQueryDataSource,
+)
+from fiddler3.schemas.xai_params import XaiParams  # noqa
+from fiddler3.utils.logger import set_logging  # noqa
+
+__all__ = [
+    'Connection',
+    'ConnectionMixin',
+    'init',
+    # Constants
+    'AlertCondition',
+    'ArtifactStatus',
+    'ArtifactType',
+    'BaselineType',
+    'BinSize',
+    'CompareTo',
+    'DataType',
+    'DeploymentType',
+    'EnvType',
+    'ExplainMethod',
+    'JobStatus',
+    'ModelInputType',
+    'ModelTask',
+    'Priority',
+    'WindowSize',
+    # Schemas
+    'CustomFeature',
+    'DatasetDataSource',
+    'DeploymentParams',
+    'Enrichment',
+    'EventIdDataSource',
+    'ImageEmbedding',
+    'ModelSchema',
+    'ModelSpec',
+    'ModelTaskParams',
+    'Multivariate',
+    'RowDataSource',
+    'SqlSliceQueryDataSource',
+    'TextEmbedding',
+    'VectorFeature',
+    'XaiParams',
+    # Entities
+    'AlertRecord',
+    'AlertRule',
+    'Baseline',
+    'Dataset',
+    'File',
+    'Job',
+    'Model',
+    'ModelDeployment',
+    'Project',
+    'Webhook',
+    # Exceptions
+    'NotFound',
+    'Conflict',
+    'IncompatibleClient',
+    'AsyncJobFailed',
+    'Unsupported',
+    'HttpError',
+    'ConnectionTimeout',
+    'ConnectionError',
+    'ApiError',
+    # Utilities
+    'set_logging',
+]

{fiddler-client-2.4.0.dev2 → fiddler-client-2.5.0.dev2}/fiddler3/configs.py

@@ -3,3 +3,6 @@ MIN_SERVER_VERSION = '23.7.0'  # @TODO: Change this before 3.0 release
 DEFAULT_PAGE_SIZE = 50
 JOB_POLL_INTERVAL = 3
 JOB_WAIT_TIMEOUT = 1800
+STREAM_EVENT_LIMIT = 1000
+DEFAULT_NUM_CLUSTERS = 5
+DEFAULT_NUM_TAGS = 5