fiddler-client 2.4.0.dev2__py3-none-any.whl → 2.5.0.dev2__py3-none-any.whl

fiddler3/entities/xai.py

@@ -9,8 +9,11 @@ from uuid import UUID
 
 import pandas as pd
 
+from fiddler3.constants.dataset import EnvType
 from fiddler3.constants.xai import ExplainMethod
 from fiddler3.decorators import handle_api_error
+from fiddler3.entities.job import Job
+from fiddler3.schemas.job import JobCompactResp
 from fiddler3.schemas.xai import (
     DatasetDataSource,
     EventIdDataSource,
@@ -18,12 +21,19 @@ from fiddler3.schemas.xai import (
     SqlSliceQueryDataSource,
 )
 from fiddler3.utils.helpers import try_series_retype
+from fiddler3.utils.logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class XaiMixin:
     id: UUID | None
     _client: Callable
 
+    def _get_method(self, update: bool = False) -> Callable:
+        """Get HTTP method"""
+        return self._client().put if update else self._client().post
+
     @handle_api_error
     def explain(  # pylint: disable=too-many-arguments
         self,
@@ -64,7 +74,7 @@ class XaiMixin:
             'explanation_type': method,
         }
         if ref_data_source:
-            payload['ref_data_source'] = ref_data_source.dict()
+            payload['ref_data_source'] = ref_data_source.dict(exclude_none=True)
         if num_permutations:
             payload['num_permutations'] = num_permutations
         if ci_level:
@@ -104,7 +114,7 @@ class XaiMixin:
 
         payload: dict[str, Any] = {
             'model_id': self.id,
-            'data_source': data_source.dict(),
+            'data_source': data_source.dict(exclude_none=True),
             'protected_features': protected_features,
             'positive_outcome': positive_outcome,
         }
@@ -183,8 +193,8 @@ class XaiMixin:
         """
         self._check_id_attributes()
         output_dir = Path(output_dir)
-        if output_dir.exists():
-            raise ValueError(f'Output dir already exists {output_dir}')
+        if not output_dir.exists():
+            os.makedirs(output_dir)
         payload: dict[str, Any] = {
             'model_id': self.id,
             'query': query,
@@ -195,7 +205,6 @@ class XaiMixin:
         if columns:
             payload['columns'] = columns
 
-        os.makedirs(output_dir)
         file_path = os.path.join(output_dir, 'output.parquet')
         with self._client().post(url='/v3/slice-query/download', data=payload) as resp:
             # Download parquet file
@@ -273,6 +282,221 @@ class XaiMixin:
         )
         return pd.DataFrame(response.json()['data']['predictions'])
 
+    @handle_api_error
+    def get_feature_impact(  # pylint: disable=too-many-arguments
+        self,
+        data_source: DatasetDataSource | SqlSliceQueryDataSource,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+        min_support: int | None = None,
+        output_columns: list[str] | None = None,
+    ) -> tuple:
+        """
+        Get global feature impact for a model over a dataset or a slice.
+
+        :param data_source: DataSource for the input dataset to compute feature
+            impact on (DatasetDataSource or SqlSliceQueryDataSource)
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+        :param min_support: Only used for NLP (TEXT inputs) models. Specify a minimum
+            support (number of times a specific word was present in the sample data)
+            to retrieve top words. Default to 15.
+        :param output_columns: Only used for NLP (TEXT inputs) models. Output column
+            names to compute feature impact on.
+
+        :return: Feature Impact tuple
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'data_source': data_source.dict(exclude_none=True),
+        }
+
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+        if min_support:
+            payload['min_support'] = min_support
+        if output_columns:
+            payload['output_columns'] = output_columns
+
+        response = self._client().post(
+            url='/v3/analytics/feature-impact',
+            data=payload,
+        )
+
+        return namedtuple('FeatureImpact', response.json()['data'])(
+            **response.json()['data']
+        )
+
+    @handle_api_error
+    def get_feature_importance(  # pylint: disable=too-many-arguments
+        self,
+        data_source: DatasetDataSource | SqlSliceQueryDataSource,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+    ) -> tuple:
+        """
+        Get global feature importance for a model over a dataset or a slice.
+
+        :param data_source: DataSource for the input dataset to compute feature
+            importance on (DatasetDataSource or SqlSliceQueryDataSource)
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+
+        :return: Feature Importance tuple
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'data_source': data_source.dict(),
+        }
+
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+
+        response = self._client().post(
+            url='/v3/analytics/feature-importance',
+            data=payload,
+        )
+
+        return namedtuple('FeatureImportance', response.json()['data'])(
+            **response.json()['data']
+        )
+
+    @handle_api_error
+    def precompute_feature_impact(  # pylint: disable=too-many-arguments
+        self,
+        dataset_id: UUID,
+        num_samples: int | None = None,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+        min_support: int | None = None,
+        update: bool = False,
+    ) -> Job:
+        """Pre-compute feature impact for a model on a dataset.
+
+        This is used in various places in the UI.
+        A single feature impact can be precomputed (computed and cached) for a model.
+
+        :param dataset_id: The unique identifier of the dataset
+        :param num_samples: The number of samples used
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+        :param min_support: Only used for NLP (TEXT inputs) models. Specify a minimum
+            support (number of times a specific word was present in the sample data)
+            to retrieve top words. Default to 15.
+        :param update: Whether the precomputed feature impact should be recomputed and updated
+
+        :return: Async Job
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_id': dataset_id,
+            'env_type': EnvType.PRE_PRODUCTION,
+        }
+        if num_samples:
+            payload['num_samples'] = num_samples
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+        if min_support:
+            payload['min_support'] = min_support
+
+        method = self._get_method(update)
+
+        response = method(
+            url='/v3/analytics/precompute-feature-impact',
+            data=payload,
+        )
+
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        logger.info(
+            'Model[%s] - Submitted job (%s) for precomputing feature impact',
+            self.id,
+            job_compact.id,
+        )
+        return Job.get(id_=job_compact.id)
+
+    @handle_api_error
+    def precompute_feature_importance(  # pylint: disable=too-many-arguments
+        self,
+        dataset_id: UUID,
+        num_samples: int | None = None,
+        num_iterations: int | None = None,
+        num_refs: int | None = None,
+        ci_level: float | None = None,
+        update: bool = False,
+    ) -> Job:
+        """Pre-compute feature importance for a model on a dataset.
+
+        This is used in various places in the UI.
+        A single feature importance can be precomputed (computed and cached) for a model.
+
+        :param dataset_id: The unique identifier of the dataset
+        :param num_samples: The number of samples used
+        :param num_iterations: The maximum number of ablated model inferences per feature
+        :param num_refs: The number of reference points used in the explanation
+        :param ci_level: The confidence level (between 0 and 1)
+        :param update: Whether the precomputed feature impact should be recomputed and updated
+
+        :return: Async Job
+        """
+
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_id': dataset_id,
+            'env_type': EnvType.PRE_PRODUCTION,
+        }
+        if num_samples:
+            payload['num_samples'] = num_samples
+        if num_refs:
+            payload['num_refs'] = num_refs
+        if num_iterations:
+            payload['num_iterations'] = num_iterations
+        if ci_level:
+            payload['ci_level'] = ci_level
+
+        method = self._get_method(update)
+
+        response = method(
+            url='/v3/analytics/precompute-feature-importance',
+            data=payload,
+        )
+
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        logger.info(
+            'Model[%s] - Submitted job (%s) for precomputing feature importance',
+            self.id,
+            job_compact.id,
+        )
+        return Job.get(id_=job_compact.id)
+
     @handle_api_error
     def get_precomputed_feature_importance(self) -> tuple:
         """Get precomputed feature importance for a model"""
@@ -301,6 +525,48 @@ class XaiMixin:
             **response.json()['data']
         )
 
+    @handle_api_error
+    def precompute_predictions(
+        self,
+        dataset_id: UUID,
+        chunk_size: int | None = None,
+        update: bool = False,
+    ) -> Job:
+        """
+        Pre-compute predictions for a model on a dataset
+
+        :param dataset_id: The unique identifier of the dataset
+        :param chunk_size: Chunk size for fetching predictions
+        :param update: Whether the pre-computed predictions should be re-computed and updated for this dataset
+
+        :return: Dataframe of the predictions
+        """
+        self._check_id_attributes()
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_id': dataset_id,
+        }
+
+        if chunk_size:
+            payload['batch_size'] = chunk_size
+
+        method = self._get_method(update)
+
+        response = method(
+            url='/v3/analytics/precompute-predictions',
+            data=payload,
+        )
+
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        logger.info(
+            'Model[%s] - Submitted job (%s) for precomputing predictions on dataset[%s]',
+            self.id,
+            job_compact.id,
+            dataset_id,
+        )
+        return Job.get(id_=job_compact.id)
+
     def _check_id_attributes(self) -> None:
         if not self.id:
             raise AttributeError(

fiddler3/libs/http_client.py

@@ -78,7 +78,7 @@ class RequestClient:
             request_headers.update(headers)
 
         content_type = request_headers.get('Content-Type')
-        if data and content_type == JSON_CONTENT_TYPE:
+        if data is not None and content_type == JSON_CONTENT_TYPE:
             data = simplejson.dumps(data, ignore_nan=True, cls=RequestClientJSONEncoder)  # type: ignore
 
         kwargs.setdefault('allow_redirects', True)

fiddler3/libs/json_encoder.py

@@ -8,5 +8,5 @@ class RequestClientJSONEncoder(JSONEncoder):
     def default(self, o: Any) -> Any:
         """Override JSONEncoder.default to support uuid serialization"""
         if isinstance(o, UUID):
-            return o.hex
+            return str(o)
         return super().default(o)

fiddler3/schemas/__init__.py

@@ -1,9 +0,0 @@
-"""Initialize all schemas."""
-from fiddler3.schemas.custom_features import *  # noqa
-from fiddler3.schemas.dataset import EnvType  # noqa
-from fiddler3.schemas.deployment_params import DeploymentParams  # noqa
-from fiddler3.schemas.model_schema import ModelSchema  # noqa
-from fiddler3.schemas.model_spec import ModelSpec  # noqa
-from fiddler3.schemas.model_task_params import ModelTaskParams  # noqa
-from fiddler3.schemas.xai import *  # noqa
-from fiddler3.schemas.xai_params import XaiParams  # noqa

fiddler3/schemas/alert_record.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from typing import Optional
+from uuid import UUID
+
+from pydantic import Field
+
+from fiddler3.schemas.base import BaseModel
+
+
+class AlertRecordResp(BaseModel):
+    id: UUID = Field(alias='uuid')
+    alert_rule_id: UUID = Field(alias='alert_config_uuid')
+    alert_run_start_time: int
+    alert_time_bucket: int
+    alert_value: float
+    baseline_time_bucket: Optional[int]
+    baseline_value: Optional[float]
+    is_alert: bool
+    severity: str
+    failure_reason: str
+    message: str
+    feature_name: Optional[str]
+    alert_record_main_version: int
+    alert_record_sub_version: int
+
+    created_at: datetime
+    updated_at: datetime

fiddler3/schemas/alert_rule.py

@@ -0,0 +1,39 @@
+from datetime import datetime
+from typing import List, Optional, Union
+from uuid import UUID
+
+from pydantic import Field
+
+from fiddler3.constants.alert_rule import AlertCondition, BinSize, CompareTo, Priority
+from fiddler3.schemas.base import BaseModel
+from fiddler3.schemas.baseline import BaselineCompactResp
+from fiddler3.schemas.model import ModelCompactResp
+from fiddler3.schemas.project import ProjectCompactResp
+
+
+class AlertRuleResp(BaseModel):
+    id: UUID = Field(alias='uuid')
+    name: str
+    model: ModelCompactResp
+    project: ProjectCompactResp
+    baseline: Optional[BaselineCompactResp]
+    priority: Union[str, Priority]
+    compare_to: Union[str, CompareTo]
+    metric_id: Union[str, UUID] = Field(alias='metric')
+    critical_threshold: float
+    condition: Union[str, AlertCondition]
+    bin_size: Union[str, BinSize]
+    columns: Optional[List[str]]
+    baseline_id: Optional[UUID]
+    compare_bin_delta: Optional[int]
+    warning_threshold: Optional[float]
+
+    created_at: datetime
+    updated_at: datetime = Field(alias='last_updated')
+
+
+class NotificationConfig(BaseModel):
+    emails: Optional[List[str]]
+    pagerduty_services: Optional[List[str]]
+    pagerduty_severity: Optional[str]
+    webhooks: Optional[List[UUID]]

fiddler3/schemas/custom_features.py

@@ -1,19 +1,11 @@
-from enum import Enum
 from typing import Any, Dict, List, Optional, Type, TypeVar
 
-from pydantic import BaseModel
-
-CustomFeatureTypeVar = TypeVar('CustomFeatureTypeVar', bound='CustomFeature')
-DEFAULT_NUM_CLUSTERS = 5
-DEFAULT_NUM_TAGS = 5
+from pydantic import BaseModel, validator
 
+from fiddler3.configs import DEFAULT_NUM_CLUSTERS, DEFAULT_NUM_TAGS
+from fiddler3.constants.model import CustomFeatureType
 
-class CustomFeatureType(str, Enum):
-    FROM_COLUMNS = 'FROM_COLUMNS'
-    FROM_VECTOR = 'FROM_VECTOR'
-    FROM_TEXT_EMBEDDING = 'FROM_TEXT_EMBEDDING'
-    FROM_IMAGE_EMBEDDING = 'FROM_IMAGE_EMBEDDING'
-    ENRICHMENT = 'ENRICHMENT'
+CustomFeatureTypeVar = TypeVar('CustomFeatureTypeVar', bound='CustomFeature')
 
 
 class CustomFeature(BaseModel):
@@ -22,6 +14,12 @@ class CustomFeature(BaseModel):
     n_clusters: Optional[int] = DEFAULT_NUM_CLUSTERS
     centroids: Optional[List] = None
 
+    @validator('n_clusters')
+    def validate_n_clusters(cls, value: int) -> int:  # noqa: N805
+        if value < 0:
+            raise ValueError('n_clusters must be greater than 0')
+        return value
+
     class Config:
         allow_mutation = False
 
@@ -46,6 +44,8 @@ class CustomFeature(BaseModel):
             return TextEmbedding.parse_obj(deserialized_json)
         if feature_type == CustomFeatureType.FROM_IMAGE_EMBEDDING:
             return ImageEmbedding.parse_obj(deserialized_json)
+        if feature_type == CustomFeatureType.ENRICHMENT:
+            return Enrichment.parse_obj(deserialized_json)
 
         raise ValueError(f'Unsupported feature type: {feature_type}')
 
@@ -63,6 +63,10 @@ class CustomFeature(BaseModel):
                 return_dict['source_column'] = self.source_column
                 if isinstance(self, TextEmbedding):
                     return_dict['n_tags'] = self.n_tags
+        elif isinstance(self, Enrichment):
+            return_dict['columns'] = self.columns
+            return_dict['enrichment'] = self.enrichment
+            return_dict['config'] = self.config
         else:
             raise ValueError(f'Unsupported feature type: {self.type} {type(self)}')
 
@@ -74,6 +78,12 @@ class Multivariate(CustomFeature):
     columns: List[str]
     monitor_components: bool = False
 
+    @validator('columns')
+    def validate_columns(cls, value: List[str]) -> List[str]:  # noqa: N805
+        if len(value) < 2:
+            raise ValueError('Multivariate columns must be greater than 1')
+        return value
+
 
 class VectorFeature(CustomFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_VECTOR
@@ -85,6 +95,52 @@ class TextEmbedding(VectorFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_TEXT_EMBEDDING
     n_tags: Optional[int] = DEFAULT_NUM_TAGS
 
+    @validator('source_column')
+    def validate_source_column(cls, value: str) -> str:  # noqa: N805
+        if value is None:
+            raise ValueError('source_column must be specified')
+        return value
+
+    @validator('n_tags')
+    def validate_n_tags(cls, value: int) -> int:  # noqa: N805
+        if value < 0:
+            raise ValueError('n_tags must be greater than 0')
+        return value
+
 
 class ImageEmbedding(VectorFeature):
     type: CustomFeatureType = CustomFeatureType.FROM_IMAGE_EMBEDDING
+
+    @validator('source_column')
+    def validate_source_column(cls, value: str) -> str:  # noqa: N805
+        if value is None:
+            raise ValueError('source_column must be specified')
+        return value
+
+
+class Enrichment(CustomFeature):
+    """
+    Represents an enrichment feature in a data processing or machine learning context.
+
+    This class inherits from CustomFeature and is used to apply specific enrichment
+    operations to a set of columns in a dataset. The type of enrichment and its
+    configuration can be specified.
+
+    Attributes:
+        type (CustomFeatureType): The type of the custom feature, set to ENRICHMENT.
+        columns (List[str]): The list of input column names in the dataset used to generate the enrichment.
+        enrichment (str): A string identifier for the type of enrichment to be applied.
+        config (Dict[str, Any]): A dictionary containing configuration options for the enrichment.
+    """
+
+    # Setting the feature type to ENRICHMENT
+    type: CustomFeatureType = CustomFeatureType.ENRICHMENT
+
+    # List of input column names used to generate the enrichment
+    columns: List[str]
+
+    # String identifier for the enrichment to be applied. e.g. "embedding" or "toxicity"
+    enrichment: str
+
+    # Dictionary for additional configuration options for the enrichment
+    config: Dict[str, Any] = {}

fiddler3/schemas/events.py

@@ -0,0 +1,15 @@
+from typing import Any, Dict, List
+from uuid import UUID
+
+from fiddler3.constants.events import PublishEventsSourceType
+from fiddler3.schemas.base import BaseModel
+
+
+class FileSource(BaseModel):
+    file_id: UUID
+    type: PublishEventsSourceType = PublishEventsSourceType.FILE
+
+
+class EventsSource(BaseModel):
+    events: List[Dict[Any, Any]]
+    type: PublishEventsSourceType = PublishEventsSourceType.EVENTS

fiddler3/schemas/file.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import Field
+
+from fiddler3.schemas.base import BaseModel
+from fiddler3.schemas.user import UserCompactResp
+
+
+class FileResp(BaseModel):
+    id: UUID
+    name: str = Field(alias='filename')
+    status: str
+    type: str
+    created_at: datetime
+    updated_at: datetime
+    created_by: UserCompactResp
+    updated_by: UserCompactResp
+
+
+class FileCompactResp(BaseModel):
+    id: UUID
+    name: str

fiddler3/schemas/model.py

@@ -5,7 +5,7 @@ from uuid import UUID
 from pydantic.fields import Field
 
 from fiddler3.schemas.base import BaseModel
-from fiddler3.schemas.model_schema import Column, ModelSchema
+from fiddler3.schemas.model_schema import ModelSchema
 from fiddler3.schemas.model_spec import ModelSpec
 from fiddler3.schemas.model_task_params import ModelTaskParams
 from fiddler3.schemas.organization import OrganizationCompactResp
@@ -40,9 +40,4 @@ class ModelResp(BaseModel):
     updated_by: UserCompactResp
     organization: OrganizationCompactResp
     project: ProjectCompactResp
-    input_cols: List[Column]
-    output_cols: List[Column]
-    target_cols: List[Column]
-    metadata_cols: List[Column]
-    decision_cols: List[Column]
     is_binary_ranking_model: bool

fiddler3/schemas/model_deployment.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import Optional
 from uuid import UUID
 
 from fiddler3.schemas.base import BaseModel
@@ -16,10 +17,10 @@ class ModelDeploymentResponse(BaseModel):
     artifact_type: str
     deployment_type: str
     active: bool
-    image_uri: str
-    replicas: int
-    cpu: int
-    memory: int
+    image_uri: Optional[str]
+    replicas: Optional[int]
+    cpu: Optional[int]
+    memory: Optional[int]
     created_at: datetime
     updated_at: datetime
     created_by: UserCompactResp

fiddler3/schemas/webhook.py

@@ -0,0 +1,23 @@
+import enum
+from datetime import datetime
+from uuid import UUID
+
+from pydantic.fields import Field
+
+from fiddler3.schemas.base import BaseModel
+
+
+@enum.unique
+class WebhookProvider(str, enum.Enum):
+    # provider is 'SLACK' or 'OTHER' as of Aug 2023.
+    SLACK = 'SLACK'
+    OTHER = 'OTHER'
+
+
+class WebhookResp(BaseModel):
+    id: UUID = Field(alias='uuid')
+    name: str
+    url: str
+    provider: WebhookProvider
+    created_at: datetime
+    updated_at: datetime