fiddler-client 3.5.0.dev2__tar.gz → 3.6.0.dev1__tar.gz

{fiddler_client-3.5.0.dev2/fiddler_client.egg-info → fiddler_client-3.6.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fiddler-client
-Version: 3.5.0.dev2
+Version: 3.6.0.dev1
 Summary: Python client for Fiddler Platform
 Home-page: https://fiddler.ai
 Author: Fiddler Labs

{fiddler_client-3.5.0.dev2 → fiddler_client-3.6.0.dev1}/README.md

@@ -1,6 +1,6 @@
-[![codecov](https://codecov.io/gh/fiddler-labs/python-client/graph/badge.svg?token=2E6AAUXO56)](https://codecov.io/gh/fiddler-labs/python-client)
+# fiddler-client
 
-# Fiddler Python client
+A Python library for interacting with the Fiddler HTTP API.
 
 ## Pre Release
 

fiddler_client-3.6.0.dev1/fiddler/VERSION

@@ -0,0 +1 @@
+3.6.0.dev1

{fiddler_client-3.5.0.dev2 → fiddler_client-3.6.0.dev1}/fiddler/__init__.py

@@ -57,7 +57,6 @@ from fiddler.schemas.xai import (  # noqa
     DatasetDataSource,
     EventIdDataSource,
     RowDataSource,
-    SqlSliceQueryDataSource,
 )
 from fiddler.schemas.xai_params import XaiParams  # noqa
 from fiddler.utils.helpers import group_by  # noqa
@@ -110,7 +109,6 @@ __all__ = [
     'ModelTaskParams',
     'Multivariate',
     'RowDataSource',
-    'SqlSliceQueryDataSource',
     'TextEmbedding',
     'VectorFeature',
     'XaiParams',

fiddler_client-3.6.0.dev1/fiddler/configs.py

@@ -0,0 +1,13 @@
+DEFAULT_CONNECTION_TIMEOUT = 60  # In seconds
+MIN_SERVER_VERSION = '24.4.0'
+DEFAULT_PAGE_SIZE = 50
+
+# In seconds. For example, this generates a log message every N seconds.
+# Was once set to 3, and I think that was too frequent. If this needs to
+# be snappier, we can grow the interval.
+JOB_POLL_INTERVAL = 6
+JOB_WAIT_TIMEOUT = 1800
+
+STREAM_EVENT_LIMIT = 1000
+DEFAULT_NUM_CLUSTERS = 5
+DEFAULT_NUM_TAGS = 5

fiddler_client-3.6.0.dev1/fiddler/decorators.py

@@ -0,0 +1,84 @@
+from functools import wraps
+from http import HTTPStatus
+from typing import Any, Callable, TypeVar, cast
+
+import requests
+import pydantic
+
+from fiddler.exceptions import ApiError, Conflict, HttpError, NotFound, Unsupported
+from fiddler.schemas.response import ErrorResponse
+from fiddler.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+_WrappedFuncType = TypeVar(  # pylint: disable=invalid-name
+    '_WrappedFuncType', bound=Callable[..., Any]
+)
+
+
+# ignore: fiddler/decorators.py:19:1: C901 'handle_api_error' is too complex (11)
+def handle_api_error(func: _WrappedFuncType) -> _WrappedFuncType:  # noqa: C901
+    @wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except requests.JSONDecodeError as e:
+            # This often is premature use of `json()`. Calling code should
+            # first check the response for the expected status code, and only
+            # then attempt JSON-deserialization.
+            raise HttpError(message=f'Invalid JSON response - {e.doc}')  # type: ignore
+        except requests.HTTPError as http_exc:
+            # Note(JP): when we're here it means we got an HTTP response.
+            # That's how this exception is documented.
+
+            status = http_exc.response.status_code
+            # The logging levels 'critical' and 'error' are generally meant to
+            # be used for terminal / fatal errors. Here, we handle many
+            # transient (retryable) errors, and it's fair to log them not as
+            # serious as actual errors. Therefore level WARNING and not ERROR.
+            # Even INFO might make sense.
+            logger.warning(
+                '%s HTTP request to %s failed with %s - %s',
+                http_exc.request.method,
+                getattr(http_exc.request, 'url', 'unknown'),
+                status,
+                # Not entire response body in case we have a megabyte response.
+                getattr(http_exc.response, 'content', 'missing')[:200],
+            )
+
+            # Now, map the error at hand to an exception to be thrown towards
+            # the caller.
+
+            if status == HTTPStatus.METHOD_NOT_ALLOWED:
+                raise Unsupported()
+
+            # Did we get structured error information? This might even
+            # be wrapped in a 501 response for example.
+            try:
+                # https://github.com/fiddler-labs/fiddler/issues/9314
+                error_resp = ErrorResponse(**http_exc.response.json())
+            except pydantic.ValidationError:
+                # We couldn't turn the HTTP error response into an object
+                # satisfying model='ErrorResponse'. Now, instead of exposing
+                # this to users as a pydantic.ValidationError (which is kinda
+                # useless to them) -- just re-raise requests.HTTPError, with
+                # all its useful error detail -- it is likely to show the
+                # actual problem.
+                raise http_exc
+            except requests.JSONDecodeError:
+                # Re-raise requests.HTTPError, with all its useful error
+                # detail.
+                raise http_exc
+
+            # This code path is only reachable if we could successfully
+            # deserialize an ErrorResponse object from the HTTP response body.
+
+            if status == HTTPStatus.CONFLICT:
+                raise Conflict(error_resp.error)
+
+            if status == HTTPStatus.NOT_FOUND:
+                raise NotFound(error_resp.error)
+
+            raise ApiError(error_resp.error)
+
+    return cast(_WrappedFuncType, wrapper)

{fiddler_client-3.5.0.dev2 → fiddler_client-3.6.0.dev1}/fiddler/entities/alert_rule.py

@@ -247,7 +247,7 @@ class AlertRule(
             url=self._get_url(id_=self.id), data={'enable_notification': True}
         )
         logger.info(
-            'Notifications have been enabled for alert rule with id: %s', self.id
+            'notifications have been enabled for alert rule with id: %s', self.id
         )
 
     @handle_api_error

{fiddler_client-3.5.0.dev2 → fiddler_client-3.6.0.dev1}/fiddler/entities/job.py

@@ -4,6 +4,8 @@ import time
 from typing import Iterator
 from uuid import UUID
 
+import requests
+
 from fiddler.configs import JOB_POLL_INTERVAL, JOB_WAIT_TIMEOUT
 from fiddler.constants.job import JobStatus
 from fiddler.decorators import handle_api_error
@@ -112,24 +114,57 @@ class Job(BaseEntity):  # pylint: disable=too-many-instance-attributes
         :return: Iterator of job objects
         """
         assert self.id is not None
+        deadline = time.monotonic() + timeout
 
-        start_time = time.time()
         while True:
+
+            if time.monotonic() > deadline:
+                raise TimeoutError(f'Deadline exceeded while watching job {self.id}')
+
             try:
-                response = self._client().get(url=self._get_url(id_=self.id))
+                # This can raise requests.HTTPError to represent non-2xx
+                # responses.
+                response = self._client().get(
+                    url=self._get_url(id_=self.id),
+                    # Short-ish TCP connect timeout, to stay responsive in
+                    # terms of logging. The HTTP response latency for GET
+                    # /jobs/<id> is expected to be less than couple of seconds
+                    # (i.e., 30 s includes lots of  leeway).
+                    timeout=(5, 30),
+                    # Inject `retry="off"` to disable the centralized retry
+                    # machinery: ask for being confronted with the details
+                    # (throw exceptions my way, so I an do my own "responsive"
+                    # type of retrying). Otherwise, we'd give up control.
+                    retry="off",
+                )
                 self._refresh_from_response(response)
-            except ConnectionError:
-                logger.exception(
-                    'Failed to connect to the server. Retrying...\n'
-                    'If this error persists, please reach out to Fiddler.'
+
+            except requests.exceptions.HTTPError as exc:
+                # Note(JP): got a non-2xx HTTP response. The main purpose of
+                # this handler is to keep going after having received a 5xx
+                # response. In this case of GETting job status even some 404s
+                # might be worth retrying. That is, it's fine to give up only
+                # after reaching the deadline even if we collect some 4xx
+                # responses along the way. Noteworthy: Receiving a 5xx response
+                # here is not an error, it's an expected scenario to be
+                # accounted for.
+                logger.info(
+                    'watch: ignore unexpected response %s (URL: %s, response body prefix: %s...)',
+                    exc.response,
+                    exc.request.url,
+                    exc.response.text[:120],
                 )
                 continue
 
-            yield self
+            except requests.exceptions.RequestException:
+                # This error is in the hierarchy _above_ `HTTPError`, and in
+                # this setup catches all errors that are _not_ a bad response:
+                # DNS error, TCP connect timeout, err during sending request,
+                # err during receiving response. Rely on the error detail to
+                # have already been logged.
+                continue
 
-            current_time = time.time()
-            if (current_time - start_time) > timeout:
-                raise TimeoutError(f'Timed out while watching job {self.id}')
+            yield self
 
             if self.status in [
                 JobStatus.SUCCESS,
@@ -153,7 +188,7 @@ class Job(BaseEntity):  # pylint: disable=too-many-instance-attributes
 
         for job in self.watch(interval=interval, timeout=timeout):
             logger.info(
-                '%s: status - %s, progress - %.2f%%',
+                '%s: %s, progress: %.1f%%',
                 log_prefix,
                 job.status,
                 job.progress,

{fiddler_client-3.5.0.dev2 → fiddler_client-3.6.0.dev1}/fiddler/entities/xai.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import os
 from collections import namedtuple
 from datetime import datetime, timezone
-from io import BytesIO
 from pathlib import Path
 from typing import Any, Callable
 from uuid import UUID
@@ -18,15 +17,10 @@ from fiddler.constants.xai import (
 )
 from fiddler.decorators import handle_api_error
 from fiddler.entities.job import Job
+from fiddler.exceptions import Unsupported
 from fiddler.schemas.job import JobCompactResp
-from fiddler.schemas.xai import (
-    DatasetDataSource,
-    EventIdDataSource,
-    RowDataSource,
-    SqlSliceQueryDataSource,
-)
+from fiddler.schemas.xai import DatasetDataSource, EventIdDataSource, RowDataSource
 from fiddler.utils.decorators import check_version
-from fiddler.utils.helpers import try_series_retype
 from fiddler.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -44,7 +38,7 @@ class XaiMixin:
     def explain(  # pylint: disable=too-many-arguments
         self,
         input_data_source: RowDataSource | EventIdDataSource,
-        ref_data_source: DatasetDataSource | SqlSliceQueryDataSource | None = None,
+        ref_data_source: DatasetDataSource | None = None,
         method: ExplainMethod | str = ExplainMethod.FIDDLER_SHAP,
         num_permutations: int | None = None,
         ci_level: float | None = None,
@@ -55,8 +49,8 @@ class XaiMixin:
 
         :param input_data_source: DataSource for the input data to compute explanation
             on (RowDataSource, EventIdDataSource)
-        :param ref_data_source: DataSource for the reference data to compute explanation
-            on (DatasetDataSource, SqlSliceQueryDataSource).
+        :param ref_data_source: Dataset data source for the reference data to compute explanation
+            on.
             Only used for non-text models and the following methods:
             'SHAP', 'FIDDLER_SHAP', 'PERMUTE', 'MEAN_RESET'
         :param method: Explanation method name. Could be your custom
@@ -102,7 +96,7 @@ class XaiMixin:
         sample: bool = False,
         max_rows: int | None = None,
         columns: list[str] | None = None,
-    ) -> pd.DataFrame:
+    ) -> None:
         """
         Fetch data with slice query.
 
@@ -111,35 +105,12 @@ class XaiMixin:
                         columns to select overriding columns selected in the query.
         :param max_rows: Number of maximum rows to fetch
         :param sample: Whether rows should be sample or not from the database
-
         :return: Dataframe of the query output
         """
-        self._check_id_attributes()
-
-        payload: dict[str, Any] = {
-            'model_id': self.id,
-            'query': query,
-            'sample': sample,
-        }
-        if max_rows:
-            payload['max_rows'] = max_rows
-        if columns:
-            payload['columns'] = columns
-
-        response = self._client().post(
-            url='/v3/slice-query/fetch',
-            data=payload,
+        raise Unsupported(
+            'This method is not supported since version 3.4. Please use `model.download_data` instead.'
         )
 
-        response_dict = response.json()['data']
-
-        column_names = response_dict['metadata']['columns']
-        dtype_strings = response_dict['metadata']['dtypes']
-        df = pd.DataFrame(response_dict['rows'], columns=column_names)
-        for column_name, dtype in zip(column_names, dtype_strings):
-            df[column_name] = try_series_retype(df[column_name], dtype)
-        return df
-
     @handle_api_error
     def download_data(  # pylint: disable=too-many-arguments
         self,
@@ -250,25 +221,9 @@ class XaiMixin:
         :param max_rows: Number of maximum rows to fetch
         :param sample: Whether rows should be sample or not from the database
         """
-        self._check_id_attributes()
-        output_dir = Path(output_dir)
-        if not output_dir.exists():
-            os.makedirs(output_dir)
-        payload: dict[str, Any] = {
-            'model_id': self.id,
-            'query': query,
-            'sample': sample,
-        }
-        if max_rows:
-            payload['max_rows'] = max_rows
-        if columns:
-            payload['columns'] = columns
-
-        file_path = os.path.join(output_dir, 'output.parquet')
-        with self._client().post(url='/v3/slice-query/download', data=payload) as resp:
-            # Download parquet file
-            df = pd.read_parquet(BytesIO(resp.content))
-            df.to_parquet(file_path)
+        raise Unsupported(
+            'This method is not supported since version 3.4. Please use `model.download_data` instead.'
+        )
 
     @handle_api_error
     def get_mutual_info(
@@ -277,7 +232,7 @@ class XaiMixin:
         column_name: str,
         num_samples: int | None = None,
         normalized: bool = False,
-    ) -> dict:
+    ) -> None:
         """
         Get mutual information.
 
@@ -290,27 +245,10 @@ class XaiMixin:
                all the variables in the dataset.
         :param num_samples: Number of samples to select for computation
         :param normalized: If set to True, it will compute Normalized Mutual Information
-
         :return: a dictionary of mutual information w.r.t the given feature
                  for each column given
         """
-        self._check_id_attributes()
-
-        payload: dict[str, Any] = {
-            'model_id': self.id,
-            'query': query,
-            'column_name': column_name,
-            'normalized': normalized,
-        }
-        if num_samples:
-            payload['num_samples'] = num_samples
-
-        response = self._client().post(
-            url='/v3/analytics/mutual-info',
-            data=payload,
-        )
-
-        return response.json()['data']
+        raise Unsupported('This method is not supported since version 3.4.')
 
     @handle_api_error
     def predict(
@@ -344,7 +282,7 @@ class XaiMixin:
     @handle_api_error
     def get_feature_impact(  # pylint: disable=too-many-arguments
         self,
-        data_source: DatasetDataSource | SqlSliceQueryDataSource,
+        data_source: DatasetDataSource,
         num_iterations: int | None = None,
         num_refs: int | None = None,
         ci_level: float | None = None,
@@ -354,8 +292,8 @@ class XaiMixin:
         """
         Get global feature impact for a model over a dataset or a slice.
 
-        :param data_source: DataSource for the input dataset to compute feature
-            impact on (DatasetDataSource or SqlSliceQueryDataSource)
+        :param data_source: Dataset data Source for the input dataset to compute feature
+            impact on
         :param num_iterations: The maximum number of ablated model inferences per feature
         :param num_refs: The number of reference points used in the explanation
         :param ci_level: The confidence level (between 0 and 1)
@@ -398,7 +336,7 @@ class XaiMixin:
     @handle_api_error
     def get_feature_importance(  # pylint: disable=too-many-arguments
         self,
-        data_source: DatasetDataSource | SqlSliceQueryDataSource,
+        data_source: DatasetDataSource,
         num_iterations: int | None = None,
         num_refs: int | None = None,
         ci_level: float | None = None,
@@ -406,8 +344,8 @@ class XaiMixin:
         """
         Get global feature importance for a model over a dataset or a slice.
 
-        :param data_source: DataSource for the input dataset to compute feature
-            importance on (DatasetDataSource or SqlSliceQueryDataSource)
+        :param data_source: Dataset data aSource for the input dataset to compute feature
+            importance on
         :param num_iterations: The maximum number of ablated model inferences per feature
         :param num_refs: The number of reference points used in the explanation
         :param ci_level: The confidence level (between 0 and 1)

{fiddler_client-3.5.0.dev2 → fiddler_client-3.6.0.dev1}/fiddler/exceptions.py

@@ -17,6 +17,8 @@ class BaseError(Exception):
         super().__init__(self.message)
 
     def __str__(self) -> str:
+        # It would be better to prepend context: like FiddlerBaseError. Note
+        # that this here is the string representation of an exception.
         return str(self.message)
 
     @property
@@ -53,17 +55,26 @@ class Unsupported(BaseError):
 
 
 class HttpError(BaseError):
-    """Base class for all HTTP errors"""
+    """Base class for all HTTP errors
+
+    Deprecated. Not thrown anymore.
+    """
 
 
 class ConnTimeout(HttpError):
-    """Connection timeout error"""
+    """Connection timeout error
+
+    Deprecated. Not thrown anymore.
+    """
 
     message = 'Request timed out while trying to reach endpoint'
 
 
 class ConnError(HttpError):
-    """Connection error"""
+    """Connection error
+
+    Deprecated. Not thrown anymore.
+    """
 
     message = 'Unable to reach the given endpoint'
 

fiddler_client-3.6.0.dev1/fiddler/libs/aws.py

@@ -0,0 +1,95 @@
+# type: ignore
+# TODO: do type-checking soon: https://github.com/fiddler-anyprem/hadron/issues/25
+
+# Note(JP): allow for type hints using strings (so that for example
+# "PartnerAppAuthProvider" can be used as a type hint w/o having to import the
+# class).
+from __future__ import annotations
+
+import os
+from collections import namedtuple
+from typing import Optional
+
+from fiddler.exceptions import BaseError
+from fiddler.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+AwsSageMakerAuthConfig = namedtuple('AwsSageMakerAuthConfig', ['arn', 'url'])
+
+
+def _read_env_for_aws_sagemaker_or_raise() -> Optional[AwsSageMakerAuthConfig]:
+    """
+
+    Raise `fiddler.exceptions.BaseError` when the environment appears to be
+    misconfigured.
+    """
+    av = os.getenv("AWS_PARTNER_APP_AUTH")
+
+    if av is None:
+        return None
+
+    if av.lower().strip() != "true":
+        return None
+
+    # User has declared intent: activate SageMaker authentication. Require
+    # other environment variables to be set.
+    keys = ("AWS_PARTNER_APP_ARN", "AWS_PARTNER_APP_URL")
+    for k in keys:
+        errmsg = f"AWS_PARTNER_APP_AUTH is set to `{av}`, but {k} is not set"
+        if os.getenv(k) is None:
+            # Future: maybe have a ConfigError type?
+            raise BaseError(errmsg)
+
+    # Future: potentially do validation for got error messages, but be sure
+    # that the validation isn't getting in the way, _ever_.
+
+    # Remove leading and trailing whitespace: that's a safe operation, and
+    # always a user/config "error". That we can correct for.
+    arn = os.getenv("AWS_PARTNER_APP_ARN").strip()  # type: ignore[union-attr]
+    url = os.getenv("AWS_PARTNER_APP_URL").strip()  # type: ignore[union-attr]
+    return AwsSageMakerAuthConfig(arn, url)
+
+
+def _import_sm_sdk(c: AwsSageMakerAuthConfig):  # -> "PartnerAppAuthProvider":
+    """
+    Return `sagemaker.PartnerAppAuthProvider` instance or raise an exception.
+    """
+    try:
+        from sagemaker import PartnerAppAuthProvider
+    except Exception as exc:
+        errmsg = (
+            f"The AWS_PARTNER_APP_* environment variables are set, but "
+            f"importing a dependency failed: {exc}"
+        )
+        raise BaseError(errmsg)
+
+    # Note(JP): can initialization of this auth provider object fail? Does it
+    # use Internet resources? Update 1: this call is expected to fail with e.g.
+    # `ValueError: Must specify a valid AWS_PARTNER_APP_ARN environment
+    # variable` when this env var is not set. Update 2: this does indeed deeper
+    # validation of the user-given config values. May throw for example
+    # "ValueError: Must specify a valid AWS_PARTNER_APP_ARN environment
+    # variable" when something non-ARN-looking has been provided. That is, we
+    # can (and should) skip validating these env var values and lave this job
+    # to the SM SDK.
+    p = PartnerAppAuthProvider()
+    logger.info("initialized AWS SageMaker authentication provider: %s", p)
+    return p
+
+
+def conditionally_init_aws_sm_auth():  # -> Optional["PartnerAppAuthProvider"]:
+    """
+    Return `None` or `PartnerAppAuthProvider` instance.
+
+    Raise an exception upon configuration error or when dependencies are
+    missing.
+    """
+    aws_sm_auth_cfg = _read_env_for_aws_sagemaker_or_raise()
+
+    logger.debug("auth config: %s", aws_sm_auth_cfg)
+    if aws_sm_auth_cfg is not None:
+        return _import_sm_sdk(aws_sm_auth_cfg)
+
+    # Just to make this expected case explicit.
+    return None