fiddler-client 3.4.0.dev1__tar.gz → 3.5.0.dev1__tar.gz

{fiddler_client-3.4.0.dev1/fiddler_client.egg-info → fiddler_client-3.5.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fiddler-client
-Version: 3.4.0.dev1
+Version: 3.5.0.dev1
 Summary: Python client for Fiddler Platform
 Home-page: https://fiddler.ai
 Author: Fiddler Labs
@@ -14,7 +14,7 @@ Requires-Dist: pip>=21.0
 Requires-Dist: requests<3
 Requires-Dist: requests-toolbelt
 Requires-Dist: pandas>=1.2.5
-Requires-Dist: pydantic<2,>=1.10.15
+Requires-Dist: pydantic>=1.10.17
 Requires-Dist: deprecated==1.2.14
 Requires-Dist: tqdm
 Requires-Dist: simplejson>=3.17.0

fiddler_client-3.5.0.dev1/fiddler/VERSION

@@ -0,0 +1 @@
+3.5.0.dev1

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/__init__.py

@@ -18,7 +18,7 @@ from fiddler.constants.model import (  # noqa
     ModelTask,
 )
 from fiddler.constants.model_deployment import ArtifactType, DeploymentType  # noqa
-from fiddler.constants.xai import ExplainMethod  # noqa
+from fiddler.constants.xai import DownloadFormat, ExplainMethod  # noqa
 from fiddler.entities.alert_record import AlertRecord  # noqa
 from fiddler.entities.alert_rule import AlertRule  # noqa
 from fiddler.entities.baseline import Baseline, BaselineCompact  # noqa
@@ -89,6 +89,7 @@ __all__ = [
     'CompareTo',
     'DataType',
     'DeploymentType',
+    'DownloadFormat',
     'EnvType',
     'ExplainMethod',
     'JobStatus',

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/connection.py

@@ -4,7 +4,11 @@ from functools import cached_property
 from uuid import UUID
 
 from fiddler.configs import DEFAULT_CONNECTION_TIMEOUT, MIN_SERVER_VERSION
-from fiddler.constants.common import CLIENT_NAME, FIDDLER_CLIENT_VERSION_HEADER
+from fiddler.constants.common import (
+    CLIENT_NAME,
+    FIDDLER_CLIENT_NAME_HEADER,
+    FIDDLER_CLIENT_VERSION_HEADER,
+)
 from fiddler.decorators import handle_api_error
 from fiddler.exceptions import IncompatibleClient
 from fiddler.libs.http_client import RequestClient
@@ -52,6 +56,7 @@ class Connection:
 
         self.request_headers = {
             'Authorization': f'Bearer {token}',
+            FIDDLER_CLIENT_NAME_HEADER: CLIENT_NAME,
             FIDDLER_CLIENT_VERSION_HEADER: __version__,
         }
 

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/constants/common.py

@@ -6,6 +6,8 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S.%f'
 # Headers
 CONTENT_TYPE_HEADER_KEY = 'Content-Type'
 JSON_CONTENT_TYPE = 'application/json'
+
+FIDDLER_CLIENT_NAME_HEADER = 'X-Fiddler-Client-Name'
 FIDDLER_CLIENT_VERSION_HEADER = 'X-Fiddler-Client-Version'
 
 # Multi-part upload

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/constants/xai.py

@@ -9,3 +9,12 @@ class ExplainMethod(str, enum.Enum):
     PERMUTE = 'PERMUTE'
     ZERO_RESET = 'ZERO_RESET'
     MEAN_RESET = 'MEAN_RESET'
+
+
+@enum.unique
+class DownloadFormat(str, enum.Enum):
+    PARQUET = 'PARQUET'
+    CSV = 'CSV'
+
+
+DEFAULT_DOWNLOAD_CHUNK_SIZE = 1000

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/entities/alert_rule.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from typing import Any, Iterator
 from uuid import UUID
 
-from pydantic import ValidationError
+from pydantic.v1 import ValidationError
 
 from fiddler.constants.alert_rule import AlertCondition, BinSize, CompareTo, Priority
 from fiddler.decorators import handle_api_error
@@ -286,7 +286,7 @@ class AlertRule(
                 pagerduty_services=pagerduty_services,
                 pagerduty_severity=pagerduty_severity,
                 webhooks=webhooks,
-            ).dict()
+            ).dict(exclude_none=True)
         except ValidationError as e:
             logger.exception('Invalid input: The format of input is not correct')
             raise e

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/entities/xai.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import os
 from collections import namedtuple
+from datetime import datetime, timezone
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Callable
@@ -10,7 +11,11 @@ from uuid import UUID
 import pandas as pd
 
 from fiddler.constants.dataset import EnvType
-from fiddler.constants.xai import ExplainMethod
+from fiddler.constants.xai import (
+    DEFAULT_DOWNLOAD_CHUNK_SIZE,
+    DownloadFormat,
+    ExplainMethod,
+)
 from fiddler.decorators import handle_api_error
 from fiddler.entities.job import Job
 from fiddler.schemas.job import JobCompactResp
@@ -20,6 +25,7 @@ from fiddler.schemas.xai import (
     RowDataSource,
     SqlSliceQueryDataSource,
 )
+from fiddler.utils.decorators import check_version
 from fiddler.utils.helpers import try_series_retype
 from fiddler.utils.logger import get_logger
 
@@ -134,6 +140,97 @@ class XaiMixin:
             df[column_name] = try_series_retype(df[column_name], dtype)
         return df
 
+    @handle_api_error
+    def download_data(  # pylint: disable=too-many-arguments
+        self,
+        output_dir: Path | str,
+        env_type: EnvType,
+        env_id: UUID | None = None,
+        start_time: datetime | None = None,
+        end_time: datetime | None = None,
+        segment_id: UUID | None = None,
+        segment_definition: UUID | None = None,
+        max_rows: int | None = None,
+        columns: list[str] | None = None,
+        chunk_size: int | None = DEFAULT_DOWNLOAD_CHUNK_SIZE,
+        fetch_vectors: bool | None = None,
+        output_format: DownloadFormat = DownloadFormat.PARQUET,
+    ) -> None:
+        """
+        Download data with a slice data configuration to PARQUET or CSV file.
+
+        :param output_dir: Path to download the file
+        :param env_type: Type of environment to query (PRODUCTION or PRE_PRODUCTION)
+        :param env_id: If PRE_PRODUCTION env selected, provide the uuid of the dataset to query
+        :param start_time: Start time to retrieve data, only for PRODUCTION env. If not time zone is indicated, we will infer UTC time zone.
+        :param start_time: End time to retrieve data, only for PRODUCTION env. If not time zone is indicated, we will infer UTC time zone.
+        :param segment_id: Optional segment UUID to query data using a saved segment associated with the model
+        :param segment_definition: Optional segment FQL definition to query data using an applied segment. This segment will not be saved to the model.
+        :param columns: Allows caller to explicitly specify list of columns to retrieve. Default to None which fetch all columns from the model.
+        :param max_rows: Number of maximum rows to fetch
+        :param chunk_size: Number of rows per chunk to download data. Default to 1000. You can increase that number for faster download if you query less than 1000 columns and don't have vector columns.
+        :param fetch_vectors: Whether the vectors columns are fetched or not. Default to False.
+        :param output_format: Format indicating if the result should be a CSV file or a PARQUET file. Default to PARQUET file.
+        """
+        self._check_id_attributes()
+
+        output_dir = Path(output_dir)
+        if not output_dir.exists():
+            os.makedirs(output_dir)
+
+        payload: dict[str, Any] = {
+            'model_id': self.id,
+            'env_type': env_type,
+            'csv': True if output_format == DownloadFormat.CSV else False,
+            'column_names': columns,
+        }
+        if env_id:
+            payload['env_id'] = env_id
+        if start_time or end_time:
+            payload['time_filter'] = {
+                'start_time': start_time.astimezone(timezone.utc).strftime(
+                    '%Y-%m-%d %H:%M:%S'
+                )
+                if start_time
+                else None,
+                'end_time': end_time.astimezone(timezone.utc).strftime(
+                    '%Y-%m-%d %H:%M:%S'
+                )
+                if end_time
+                else None,
+                'time_zone': 'UTC',
+            }
+
+        if segment_id or segment_definition:
+            payload['segment'] = {'id': segment_id, 'definition': segment_definition}
+
+        if max_rows:
+            payload['num_samples'] = max_rows
+
+        if chunk_size:
+            payload['chunk_size'] = chunk_size
+
+        if fetch_vectors:
+            payload['fetch_vectors'] = fetch_vectors
+
+        file_path = os.path.join(
+            output_dir,
+            'output.csv' if output_format == DownloadFormat.CSV else 'output.parquet',
+        )
+
+        with self._client().post(
+            url='/v3/analytics/download-slice-data', data=payload
+        ) as resp:
+            # Download file
+            with open(file_path, 'wb') as f:
+                for chunk in resp.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        f.write(chunk)
+
+        logger.info(
+            'Data succesfully downloaded',
+        )
+
     @handle_api_error
     def download_slice(  # pylint: disable=too-many-arguments
         self,
@@ -536,6 +633,7 @@ class XaiMixin:
                 'API response.'
             )
 
+    @check_version(version_expr='>=24.11.1')
     @handle_api_error
     def upload_feature_impact(
         self, feature_impact_map: dict, update: bool = False

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/exceptions.py

@@ -34,8 +34,8 @@ class IncompatibleClient(BaseError):
         # @TODO - Add link to compatibility matrix doc
     )
 
-    def __init__(self, server_version: str) -> None:
-        self.message = self.message.format(
+    def __init__(self, server_version: str, message: str | None = None) -> None:
+        self.message = message or self.message.format(
             client_version=__version__, server_version=server_version
         )
 

fiddler_client-3.5.0.dev1/fiddler/libs/aws.py

@@ -0,0 +1,95 @@
+# type: ignore
+# TODO: do type-checking soon: https://github.com/fiddler-anyprem/hadron/issues/25
+
+# Note(JP): allow for type hints using strings (so that for example
+# "PartnerAppAuthProvider" can be used as a type hint w/o having to import the
+# class).
+from __future__ import annotations
+
+import os
+from collections import namedtuple
+from typing import Optional
+
+from fiddler.exceptions import BaseError
+from fiddler.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+AwsSageMakerAuthConfig = namedtuple('AwsSageMakerAuthConfig', ['arn', 'url'])
+
+
+def _read_env_for_aws_sagemaker_or_raise() -> Optional[AwsSageMakerAuthConfig]:
+    """
+
+    Raise `fiddler.exceptions.BaseError` when the environment appears to be
+    misconfigured.
+    """
+    av = os.getenv("AWS_PARTNER_APP_AUTH")
+
+    if av is None:
+        return None
+
+    if av.lower().strip() != "true":
+        return None
+
+    # User has declared intent: activate SageMaker authentication. Require
+    # other environment variables to be set.
+    keys = ("AWS_PARTNER_APP_ARN", "AWS_PARTNER_APP_URL")
+    for k in keys:
+        errmsg = f"AWS_PARTNER_APP_AUTH is set to `{av}`, but {k} is not set"
+        if os.getenv(k) is None:
+            # Future: maybe have a ConfigError type?
+            raise BaseError(errmsg)
+
+    # Future: potentially do validation for got error messages, but be sure
+    # that the validation isn't getting in the way, _ever_.
+
+    # Remove leading and trailing whitespace: that's a safe operation, and
+    # always a user/config "error". That we can correct for.
+    arn = os.getenv("AWS_PARTNER_APP_ARN").strip()  # type: ignore[union-attr]
+    url = os.getenv("AWS_PARTNER_APP_URL").strip()  # type: ignore[union-attr]
+    return AwsSageMakerAuthConfig(arn, url)
+
+
+def _import_sm_sdk(c: AwsSageMakerAuthConfig):  # -> "PartnerAppAuthProvider":
+    """
+    Return `sagemaker.PartnerAppAuthProvider` instance or raise an exception.
+    """
+    try:
+        from sagemaker import PartnerAppAuthProvider
+    except Exception as exc:
+        errmsg = (
+            f"The AWS_PARTNER_APP_* environment variables are set, but "
+            f"importing a dependency failed: {exc}"
+        )
+        raise BaseError(errmsg)
+
+    # Note(JP): can initialization of this auth provider object fail? Does it
+    # use Internet resources? Update 1: this call is expected to fail with e.g.
+    # `ValueError: Must specify a valid AWS_PARTNER_APP_ARN environment
+    # variable` when this env var is not set. Update 2: this does indeed deeper
+    # validation of the user-given config values. May throw for example
+    # "ValueError: Must specify a valid AWS_PARTNER_APP_ARN environment
+    # variable" when something non-ARN-looking has been provided. That is, we
+    # can (and should) skip validating these env var values and lave this job
+    # to the SM SDK.
+    p = PartnerAppAuthProvider()
+    logger.info("initialized AWS SageMaker authentication provider: %s", p)
+    return p
+
+
+def conditionally_init_aws_sm_auth():  # -> Optional["PartnerAppAuthProvider"]:
+    """
+    Return `None` or `PartnerAppAuthProvider` instance.
+
+    Raise an exception upon configuration error or when dependencies are
+    missing.
+    """
+    aws_sm_auth_cfg = _read_env_for_aws_sagemaker_or_raise()
+
+    logger.debug("auth config: %s", aws_sm_auth_cfg)
+    if aws_sm_auth_cfg is not None:
+        return _import_sm_sdk(aws_sm_auth_cfg)
+
+    # Just to make this expected case explicit.
+    return None

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/libs/http_client.py

@@ -8,6 +8,7 @@ import requests
 import simplejson
 from requests.adapters import HTTPAdapter
 
+import fiddler.libs.aws
 from fiddler.constants.common import JSON_CONTENT_TYPE
 from fiddler.exceptions import (  # pylint: disable=redefined-builtin
     ConnError,
@@ -20,6 +21,15 @@ from fiddler.utils.logger import get_logger
 logger = get_logger(__name__)
 
 
+# Note(JP): this is doing conditional runtime modification for AWS Hadron /
+# SageMaker, depending on
+# - environment variables set by the user
+# - importability of the AWS sagemaker Python SDK
+# TODO: type annotation w/o having to import PartnerAppAuthProvider? This thing
+# is either None or an instantiated auth provider.
+_AWS_SM_AUTH_PROVIDER = fiddler.libs.aws.conditionally_init_aws_sm_auth()  # type: ignore
+
+
 class RequestClient:
     def __init__(
         self,
@@ -34,11 +44,23 @@ class RequestClient:
         self.headers = headers
         self.headers.update({'Content-Type': JSON_CONTENT_TYPE})
         self.session = requests.Session()
+
+        # Note(JP): from the AWS SageMaker partner app guide.
+        if _AWS_SM_AUTH_PROVIDER is not None:
+            # Get callback class (`RequestsAuth` type`) from SM auth provider,
+            # and decorate the `requests` session object with that. This is
+            # enabling the magic of automatically mutating request headers.
+            # Among others, this injects the SigV4 header.
+            self.session.auth = _AWS_SM_AUTH_PROVIDER.get_auth()
+
         self.session.verify = verify
         adapter = HTTPAdapter(
             pool_connections=25,
             pool_maxsize=25,
         )
+
+        # Does mounting the custom HTTP adapter revert the session.auth
+        # change from above?
         self.session.mount(self.base_url, adapter)
 
     def call(

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/packtools/project_attributions_helpers.py

@@ -159,9 +159,9 @@ class IGTextAttributionsTF2Keras:
                 att, word_tokens, tokenizer, embedding_name, feature_label
             )
             gem_container = gem.GEMContainer(contents=[gem_text])
-            explanations_by_output[
-                self.output_cols[output_field_index]
-            ] = gem_container.render()
+            explanations_by_output[self.output_cols[output_field_index]] = (
+                gem_container.render()
+            )
 
         return explanations_by_output
 
@@ -195,10 +195,12 @@ class IGTabularAttributions:
                 'Parameter attr_input_names_mapping cannot be empty when IG is enabled.'
             )
         for output_field_index, att in enumerate(attributions):
-            explanations_by_output[
-                self.output_cols[output_field_index]
-            ] = get_tabular_attributions_for_output(
-                self.input_df, att, attr_input_names_mapping=attr_input_names_mapping
+            explanations_by_output[self.output_cols[output_field_index]] = (
+                get_tabular_attributions_for_output(
+                    self.input_df,
+                    att,
+                    attr_input_names_mapping=attr_input_names_mapping,
+                )
             )
         return explanations_by_output
 
@@ -386,7 +388,7 @@ def xgb_shap_transform_scale(
     base_value = 1 / (1 + np.exp(-untransformed_base_value))
 
     # Computing the original_explanation_distance to construct the distance_coefficient later on
-    original_explanation_distance = np.sum(shap_values)
+    original_explanation_distance = np.sum(shap_values)  # type: ignore
 
     # Computing the distance between the model_prediction and the transformed base_value
     distance_to_explain = model_prediction - base_value

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/alert_record.py

@@ -2,7 +2,7 @@ from datetime import datetime
 from typing import Optional
 from uuid import UUID
 
-from pydantic import Field
+from pydantic.v1 import Field
 
 from fiddler.schemas.base import BaseModel
 

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/alert_rule.py

@@ -2,7 +2,7 @@ from datetime import datetime
 from typing import List, Optional, Union
 from uuid import UUID
 
-from pydantic import Field
+from pydantic.v1 import Field
 
 from fiddler.constants.alert_rule import AlertCondition, BinSize, CompareTo, Priority
 from fiddler.schemas.base import BaseModel

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/base.py

@@ -1,5 +1,5 @@
-from pydantic import BaseModel as _BaseModel
-from pydantic import Extra
+from pydantic.v1 import BaseModel as _BaseModel
+from pydantic.v1 import Extra
 
 
 class BaseModel(_BaseModel):

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/baseline.py

@@ -2,7 +2,7 @@ from datetime import datetime
 from typing import Optional
 from uuid import UUID
 
-from pydantic import Field
+from pydantic.v1 import Field
 
 from fiddler.schemas.base import BaseModel
 from fiddler.schemas.dataset import DatasetCompactResp

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/custom_features.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict, List, Optional, Type, TypeVar, Literal
 
-from pydantic import BaseModel, validator
+from pydantic.v1 import BaseModel, validator
 
 from fiddler.configs import DEFAULT_NUM_CLUSTERS, DEFAULT_NUM_TAGS
 from fiddler.constants.model import CustomFeatureType

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/file.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from uuid import UUID
 
-from pydantic import Field
+from pydantic.v1 import Field
 
 from fiddler.schemas.base import BaseModel
 from fiddler.schemas.user import UserCompactResp

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/model_schema.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Union
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 from fiddler.constants.model import DataType
 

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/model_spec.py

@@ -1,8 +1,14 @@
 from typing import List, Union
 
-from pydantic import BaseModel, Field
+from pydantic.v1 import BaseModel, Field
 
-from fiddler.schemas.custom_features import Multivariate, VectorFeature, TextEmbedding, ImageEmbedding, Enrichment
+from fiddler.schemas.custom_features import (
+    Multivariate,
+    VectorFeature,
+    TextEmbedding,
+    ImageEmbedding,
+    Enrichment,
+)
 
 
 class ModelSpec(BaseModel):
@@ -26,7 +32,7 @@ class ModelSpec(BaseModel):
     metadata: List[str] = Field(default_factory=list)
     """Metadata columns"""
 
-    custom_features: List[Union[Multivariate, VectorFeature, TextEmbedding, ImageEmbedding, Enrichment]] = Field(
-        default_factory=list
-    )
+    custom_features: List[
+        Union[Multivariate, VectorFeature, TextEmbedding, ImageEmbedding, Enrichment]
+    ] = Field(default_factory=list)
     """Custom feature definitions"""

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/user.py

@@ -1,6 +1,6 @@
 from uuid import UUID
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 
 class UserCompactResp(BaseModel):

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/xai.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional, Union
 from uuid import UUID
 
-from pydantic import Field
+from pydantic.v1 import Field
 
 from fiddler.constants.dataset import EnvType
 from fiddler.schemas.base import BaseModel

{fiddler_client-3.4.0.dev1 → fiddler_client-3.5.0.dev1}/fiddler/schemas/xai_params.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 
 class XaiParams(BaseModel):