fiddler-client 2.1.0.dev3__tar.gz → 2.1.0.dev5__tar.gz

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fiddler-client
-Version: 2.1.0.dev3
+Version: 2.1.0.dev5
 Summary: Python client for Fiddler Service
 Home-page: https://fiddler.ai
 Author: Fiddler Labs
@@ -31,6 +31,9 @@ Description: Fiddler Client
          - #### **New Features**
           - Introduce Model Tasks `NOT_SET` and `LLM`
           - Relax Target / Output specification for model tasks `NOT_SET` and `LLM`
+          - Support custom metrics
+         - #### **Modifications**
+          - `DatasetDataSource` and `EventIdDataSource` will take `dataset_id` instead of `dataset_name`
         
         ### 2.0.4
           - #### **Modifications**

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/PUBLIC.md

@@ -24,6 +24,9 @@ Version History
  - #### **New Features**
   - Introduce Model Tasks `NOT_SET` and `LLM`
   - Relax Target / Output specification for model tasks `NOT_SET` and `LLM`
+  - Support custom metrics
+ - #### **Modifications**
+  - `DatasetDataSource` and `EventIdDataSource` will take `dataset_id` instead of `dataset_name`
 
 ### 2.0.4
   - #### **Modifications**

fiddler-client-2.1.0.dev5/fiddler/_version.py

@@ -0,0 +1 @@
+__version__ = '2.1.0.dev5'

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/fiddler/api/custom_metrics_mixin.py

@@ -1,5 +1,5 @@
 from http import HTTPStatus
-from typing import List
+from typing import List, Optional
 
 from pydantic import parse_obj_as
 
@@ -22,7 +22,12 @@ class CustomMetricsMixin:
 
     @handle_api_error_response
     def add_custom_metric(
-        self, name: str, project_id: str, model_id: str, fql: str
+        self,
+        name: str,
+        project_id: str,
+        model_id: str,
+        definition: str,
+        description: Optional[str] = None,
     ) -> CustomMetric:
         """
         Add a new `Custom Metric`.
@@ -30,7 +35,8 @@ class CustomMetricsMixin:
         :params name: name of custom metric
         :params project_id: name of project
         :params model_id: name of model
-        :params fql: The FQL expression describing the metric
+        :params definition: The FQL expression describing the metric
+        :params description: description of custom metric
 
         :returns: Created `CustomMetric` object.
         """
@@ -39,7 +45,8 @@ class CustomMetricsMixin:
             'project_name': project_id,
             'model_name': model_id,
             'name': name,
-            'fql': fql,
+            'definition': definition,
+            'description': description,
         }
         response = self.client.post(
             url='custom-metrics',
@@ -51,7 +58,7 @@ class CustomMetricsMixin:
         return CustomMetric.parse_obj(attrs)
 
     @handle_api_error_response
-    def delete_custom_metric(self, metric_id: str = None) -> None:
+    def delete_custom_metric(self, metric_id: str) -> None:
         """
         Delete a custom metric
 
@@ -95,3 +102,15 @@ class CustomMetricsMixin:
         )
         items = PaginatedResponseHandler(response).get_pagination_items()
         return parse_obj_as(List[CustomMetric], items)
+
+    @handle_api_error_response
+    def get_custom_metric(self, metric_id: str) -> CustomMetric:
+        """
+        Get a `CustomMetric` object.
+
+        :params metric_id: id of the custom metric
+
+        :returns: a `CustomMetric` object
+        """
+        response = self.client.get(url=f'custom-metrics/{metric_id}')
+        return CustomMetric.deserialize(APIResponseHandler(response))

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/fiddler/api/explainability_mixin.py

@@ -2,7 +2,7 @@ from collections import namedtuple
 from typing import Any, Dict, List, NamedTuple, Optional, Union
 
 import pandas as pd
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from fiddler.libs.http_client import RequestClient
 from fiddler.utils.decorators import handle_api_error_response
@@ -15,7 +15,7 @@ logger = get_logger(__name__)
 
 class DatasetDataSource(BaseModel):
     source_type = 'DATASET'
-    dataset_name: str
+    dataset_name: str = Field(alias='dataset_id')
     source: Optional[str]
     num_samples: Optional[int]
 
@@ -34,7 +34,7 @@ class RowDataSource(BaseModel):
 class EventIdDataSource(BaseModel):
     source_type = 'EVENT_ID'
     event_id: str
-    dataset_name: str
+    dataset_name: str = Field(alias='dataset_id')
 
 
 class ExplainabilityMixin:

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/fiddler/exceptions.py

@@ -23,7 +23,7 @@ class AsyncJobFailed(Exception):
 class ErrorResponseHandler:
     def __init__(self, http_error: HTTPError) -> None:
         self.http_error = http_error
-        self.response = http_error.response
+        self.response = getattr(http_error, 'response')
         self.ErrorResponse = namedtuple(  # type: ignore
             'ErrorResponse', ['status_code', 'error_code', 'message', 'errors']
         )

fiddler-client-2.1.0.dev5/fiddler/schema/custom_metric.py

@@ -0,0 +1,18 @@
+from typing import Optional
+
+from fiddler.schema.base import BaseDataSchema
+from fiddler.schema.user import UserCompact
+from datetime import datetime
+
+
+class CustomMetric(BaseDataSchema):
+    id: str
+    name: str
+    project_name: str
+    organization_name: str
+    name: str
+    definition: str
+    description: Optional[str]
+    created_at: Optional[datetime]
+    created_by: Optional[UserCompact]
+

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/fiddler_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fiddler-client
-Version: 2.1.0.dev3
+Version: 2.1.0.dev5
 Summary: Python client for Fiddler Service
 Home-page: https://fiddler.ai
 Author: Fiddler Labs
@@ -31,6 +31,9 @@ Description: Fiddler Client
          - #### **New Features**
           - Introduce Model Tasks `NOT_SET` and `LLM`
           - Relax Target / Output specification for model tasks `NOT_SET` and `LLM`
+          - Support custom metrics
+         - #### **Modifications**
+          - `DatasetDataSource` and `EventIdDataSource` will take `dataset_id` instead of `dataset_name`
         
         ### 2.0.4
           - #### **Modifications**

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/tests/fiddler/test_custom_metrics.py

@@ -27,17 +27,32 @@ class TestCustomMetric(BaseTestCase):
                         "project_name": "bank_churn",
                         "name": "tp_0.8",
                         "model_name": "churn_classifier",
-                        "fql": "sum(age)",
+                        "definition": "sum(age)",
                         "id": "fbba0596-1d49-4159-a5ea-e1c2f57c9c32",
-                        "organization_name": "test_organization"
+                        "organization_name": "test_organization",
+                        "description": "test description",
+                        "extra_field_from_future": "should_not_break",
+                        "created_at": "2021-03-03T18:30:00.000Z",
+                        "created_by": {
+                            "id": "1",
+                            "full_name": "test user",
+                            "email": "testuser@fiddler.ai"
+                        }
                     },
                     {
                         "project_name": "bank_churn",
                         "name": "tp_0.6",
                         "model_name": "churn_classifier",
-                        "fql": "sum(credit_score)",
+                        "definition": "sum(credit_score)",
                         "id": "3a352397-a89a-495d-a095-440f1d21a58a",
-                        "organization_name": "test_organization"
+                        "organization_name": "test_organization",
+                        "description": "test description",
+                        "created_at": "2021-03-03T18:30:00.000Z",
+                        "created_by": {
+                            "id": "2",
+                            "full_name": "test user",
+                            "email": "testuser@fiddler.ai"
+                        }
                     }
                 ]
             },
@@ -65,18 +80,32 @@ class TestCustomMetric(BaseTestCase):
             CustomMetric(
                 id='fbba0596-1d49-4159-a5ea-e1c2f57c9c32',
                 name='tp_0.8',
-                fql='sum(age)',
+                definition='sum(age)',
                 organization_name='test_organization',
                 project_name='bank_churn',
                 model_name='churn_classifier',
+                description='test description',
+                created_at='2021-03-03T18:30:00.000Z',
+                created_by={
+                    "id": "1",
+                    "full_name": "test user",
+                    "email": "testuser@fiddler.ai"
+                }
             ),
             CustomMetric(
                 id='3a352397-a89a-495d-a095-440f1d21a58a',
                 name='tp_0.6',
-                fql='sum(credit_score)',
+                definition='sum(credit_score)',
                 organization_name='test_organization',
                 project_name='bank_churn',
                 model_name='churn_classifier',
+                description='test description',
+                created_at='2021-03-03T18:30:00.000Z',
+                created_by={
+                    "id": "2",
+                    "full_name": "test user",
+                    "email": "testuser@fiddler.ai"
+                }
             ),
         ]
 
@@ -95,17 +124,19 @@ class TestCustomMetric(BaseTestCase):
 
         custom_metric = self.client.add_custom_metric(
             name='my custom',
-            fql='sum(age)',
+            definition='sum(age)',
             project_id='bank_churn',
             model_id='churn_classifier',
+            description='test description',
         )
         assert custom_metric == CustomMetric(
             id='fbba0596-1d49-4159-a5ea-e1c2f57c9c32',
             name='my custom',
-            fql='sum(age)',
+            definition='sum(age)',
             organization_name='test_organization',
             project_name='bank_churn',
             model_name='churn_classifier',
+            description='test description',
         )
 
     def test_add_custom_metric_error(self):
@@ -131,7 +162,7 @@ class TestCustomMetric(BaseTestCase):
         with self.assertRaises(BadRequest) as context:
             self.client.add_custom_metric(
                 name='my custom',
-                fql='sum(unknown)',
+                definition='sum(unknown)',
                 project_id='bank_churn',
                 model_id='churn_classifier',
             )
@@ -140,6 +171,50 @@ class TestCustomMetric(BaseTestCase):
             "Column unknown not found in model churn_classifier"
         )
 
+    def test_get_custom_metric(self):
+        uuid = 'fbba0596-1d49-4159-a5ea-e1c2f57c9c32'
+        response = {
+            "data": {
+                "project_name": "bank_churn",
+                "name": "tp_0.8",
+                "model_name": "churn_classifier",
+                "definition": "sum(age)",
+                "id": "fbba0596-1d49-4159-a5ea-e1c2f57c9c32",
+                "organization_name": "test_organization",
+                "description": "test description",
+                "extra_field_from_future": "should_not_break",
+                "created_at": "2021-03-03T18:30:00.000Z",
+                "created_by": {
+                    "id": "1",
+                    "full_name": "test user",
+                    "email": "testuser@fiddler.ai"
+                }
+            },
+            "api_version": "2.0",
+            "kind": "NORMAL"
+        }
+        url = self._get_url(uuid)
+
+        self.requests_mock.get(url, json=response)
+        cm = self.client.get_custom_metric(
+            metric_id='fbba0596-1d49-4159-a5ea-e1c2f57c9c32'
+        )
+        assert cm == CustomMetric(
+            id='fbba0596-1d49-4159-a5ea-e1c2f57c9c32',
+            name='tp_0.8',
+            definition='sum(age)',
+            organization_name='test_organization',
+            project_name='bank_churn',
+            model_name='churn_classifier',
+            description='test description',
+            created_at='2021-03-03T18:30:00.000Z',
+            created_by={
+                "id": "1",
+                "full_name": "test user",
+                "email": "testuser@fiddler.ai"
+            }
+        )
+
     def test_delete_custom_metric(self):
         uuid = 'fbba0596-1d49-4159-a5ea-e1c2f57c9c32'
         url = self._get_url(uuid)

{fiddler-client-2.1.0.dev3 → fiddler-client-2.1.0.dev5}/tests/fiddler/test_explainability_apis.py

@@ -2,9 +2,11 @@ import json
 import unittest
 
 import pandas as pd
+import pytest
 
 from fiddler.api.explainability_mixin import (
     DatasetDataSource,
+    EventIdDataSource,
     RowDataSource,
     SqlSliceQueryDataSource,
 )
@@ -19,6 +21,16 @@ class TestExplainabilityApis(BaseTestCase):
         self._dataset_name = 'test_dataset'
         self._model_name = 'test_model'
 
+    def test_pydantic_models(self):
+        dataset_source = DatasetDataSource(dataset_id='test_dataset')
+        assert dataset_source.dataset_name == 'test_dataset'
+        assert dataset_source.source_type == 'DATASET'
+
+        event_id_ds = EventIdDataSource(event_id='event_id', dataset_id='test_dataset')
+        assert event_id_ds.dataset_name == 'test_dataset'
+        assert event_id_ds.event_id == 'event_id'
+        assert event_id_ds.source_type == 'EVENT_ID'
+
     def test_feature_importance_from_dataset(self):
         url = f'{self._url}/feature-importance'
 
@@ -29,7 +41,7 @@ class TestExplainabilityApis(BaseTestCase):
             model_id=self._model_name,
             project_id=self._project_name,
             data_source=DatasetDataSource(
-                dataset_name='test_dataset', num_samples=None
+                dataset_id='test_dataset', num_samples=None
             ),
             num_refs=None,
             num_iterations=None,
@@ -95,7 +107,7 @@ class TestExplainabilityApis(BaseTestCase):
             model_id=self._model_name,
             project_id=self._project_name,
             data_source=DatasetDataSource(
-                dataset_name='test_dataset',
+                dataset_id='test_dataset',
                 num_samples=num_samples,
             ),
             num_iterations=num_iterations,
@@ -130,7 +142,7 @@ class TestExplainabilityApis(BaseTestCase):
             model_id=self._model_name,
             project_id=self._project_name,
             data_source=DatasetDataSource(
-                dataset_name='test_dataset',
+                dataset_id='test_dataset',
                 num_samples=None,
             ),
             output_columns=None,
@@ -203,7 +215,7 @@ class TestExplainabilityApis(BaseTestCase):
             model_id=self._model_name,
             project_id=self._project_name,
             data_source=DatasetDataSource(
-                dataset_name='test_dataset',
+                dataset_id='test_dataset',
                 num_samples=num_samples,
             ),
             output_columns=output_columns,
@@ -243,12 +255,12 @@ class TestExplainabilityApis(BaseTestCase):
             project_id=self._project_name,
             input_data_source=RowDataSource(row={'col_a': 3, 'col_b': 'x'}),
             ref_data_source=DatasetDataSource(
-                dataset_name=self._dataset_name, source=None, num_samples=10
+                dataset_id=self._dataset_name, source=None, num_samples=10
             ),
             explanation_type='FIDDLER_SHAP',
             num_permutations=300,
             ci_level=0.95,
-            top_n_class=None,
+            top_n_class=1,
         )
 
         request_body = json.loads(self.requests_mock.calls[0].request.body)
@@ -265,6 +277,7 @@ class TestExplainabilityApis(BaseTestCase):
         self.assertEqual(request_body['input_data_source']['source_type'], 'ROW')
         self.assertEqual(request_body['explanation_type'], 'FIDDLER_SHAP')
         self.assertEqual(request_body['num_permutations'], 300)
+        self.assertEqual(request_body['top_n_class'], 1)
 
     def test_fairness(self):
         url = f'{self._url}/fairness'
@@ -276,7 +289,7 @@ class TestExplainabilityApis(BaseTestCase):
             model_id=self._model_name,
             project_id=self._project_name,
             data_source=DatasetDataSource(
-                dataset_name=self._dataset_name, source=None, num_samples=None
+                dataset_id=self._dataset_name, source=None, num_samples=None
             ),
             protected_features=['Gender', 'Race'],
             positive_outcome='Churned',
@@ -334,6 +347,15 @@ class TestExplainabilityApis(BaseTestCase):
 
         mock_response = get_base_api_response_body(data=data)
         self.requests_mock.post(url, json=mock_response)
+
+        with pytest.raises(ValueError):
+            self.client.get_predictions(
+            project_id=self._project_name,
+            model_id=self._model_name,
+            input_df={},
+            chunk_size=10,
+        )
+
         df = pd.DataFrame([{'col1': 4, 'col2': 'xyz'}])
         self.client.get_predictions(
             project_id=self._project_name,
@@ -358,6 +380,15 @@ class TestExplainabilityApis(BaseTestCase):
 
         slice_query = 'SELECT * FROM table'  # nosec
 
+        with pytest.raises(ValueError):
+            self.client.get_mutual_information(
+                project_id=self._project_name,
+                dataset_id=self._dataset_name,
+                query=slice_query,
+                column_name=1,
+                normalized=True,
+            )
+
         self.client.get_mutual_information(
             project_id=self._project_name,
             dataset_id=self._dataset_name,

fiddler-client-2.1.0.dev3/fiddler/_version.py

@@ -1 +0,0 @@
-__version__ = '2.1.0.dev3'

fiddler-client-2.1.0.dev3/fiddler/schema/custom_metric.py

@@ -1,10 +0,0 @@
-from fiddler.schema.base import BaseDataSchema
-
-
-class CustomMetric(BaseDataSchema):
-    id: str
-    name: str
-    project_name: str
-    organization_name: str
-    name: str
-    fql: str