fiddler-client 2.4.0.dev2__py3-none-any.whl → 2.5.0.dev2__py3-none-any.whl

fiddler3/entities/events.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterator
+from uuid import UUID
+
+import pandas as pd
+from requests import Response
+
+from fiddler3.configs import STREAM_EVENT_LIMIT
+from fiddler3.connection import ConnectionMixin
+from fiddler3.decorators import handle_api_error
+from fiddler3.entities.file import File
+from fiddler3.entities.job import Job
+from fiddler3.schemas.dataset import EnvType
+from fiddler3.schemas.events import EventsSource, FileSource
+from fiddler3.schemas.job import JobCompactResp
+
+
+class EventPublisher(ConnectionMixin):
+    STREAM_LIMIT = STREAM_EVENT_LIMIT
+
+    def __init__(self, model_id: UUID) -> None:
+        """
+        Event publishing methods
+
+        :param model_id: Model identifier
+        """
+        self.model_id = model_id
+
+    def _get_stream_chunks(
+        self, source: pd.DataFrame | list[dict[str, Any]]
+    ) -> Iterator[list[dict[str, Any]]]:
+        """Chunk the source based on stream limit"""
+
+        for i in range(0, len(source), self.STREAM_LIMIT):
+            chunk = source[i : i + self.STREAM_LIMIT]
+
+            if isinstance(chunk, pd.DataFrame):
+                events = chunk.to_dict('records')
+            else:
+                events = chunk
+
+            yield events
+
+    @handle_api_error
+    def publish(
+        self,
+        source: list[dict[str, Any]] | str | Path | pd.DataFrame,
+        environment: EnvType = EnvType.PRODUCTION,
+        dataset_name: str | None = None,
+        update: bool = False,
+    ) -> list[UUID] | Job:
+        """
+        Publish Pre-production or Production data
+
+        :param source: one of:
+            Path or str path: path for data file.
+            list[dict]: list of event dicts (max 1000) EnvType.PRE_PRODUCTION is not supported
+            dataframe: events dataframe. EnvType.PRE_PRODUCTION not supported.
+        :param environment: Either EnvType.PRE_PRODUCTION or EnvType.PRODUCTION
+        :param dataset_name: Name of the dataset. Not supported for EnvType.PRODUCTION
+        :param update: flag indicating if the events are updates to previously published rows
+
+        :return: list[UUID] for list of dicts or dataframe source and Job object for file path source.
+        """
+        if isinstance(source, (str, Path)):
+            return self._publish_file(
+                source=source,
+                environment=environment,
+                dataset_name=dataset_name,
+                update=update,
+            )
+
+        if isinstance(source, (list, pd.DataFrame)):
+            return self._publish_stream(source=source, update=update)
+
+        raise ValueError(f'Unsupported source - {type(source)}')
+
+    def _publish_stream(
+        self,
+        source: list[dict[str, Any]] | pd.DataFrame,
+        update: bool = False,
+    ) -> list[UUID]:
+        fiddler_ids = []
+        for events in self._get_stream_chunks(source=source):
+            response = self._publish_call(
+                source=EventsSource(events=events),
+                environment=EnvType.PRODUCTION,
+                dataset_name=None,
+                update=update,
+            )
+            fiddler_ids.extend(response.json()['data']['fiddler_ids'])
+
+        return fiddler_ids
+
+    def _publish_file(
+        self,
+        source: Path | str,
+        environment: EnvType,
+        dataset_name: str | None = None,
+        update: bool = False,
+    ) -> Job:
+        file = File(path=Path(source)).upload()
+
+        assert file.id is not None
+
+        response = self._publish_call(
+            source=FileSource(file_id=file.id),
+            environment=environment,
+            dataset_name=dataset_name,
+            update=update,
+        )
+        job_compact = JobCompactResp(**response.json()['data']['job'])
+        return Job.get(id_=job_compact.id)
+
+    def _publish_call(
+        self,
+        source: FileSource | EventsSource,
+        environment: EnvType,
+        dataset_name: str | None = None,
+        update: bool = False,
+    ) -> Response:
+        if update:
+            method = self._client().patch
+        else:
+            method = self._client().post
+        return method(
+            url='/v3/events',
+            data={
+                'source': source.dict(),
+                'model_id': self.model_id,
+                'env_type': environment,
+                'env_name': dataset_name,
+            },
+        )

fiddler3/entities/file.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import io
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from uuid import UUID
+
+from requests import Response
+from requests_toolbelt import MultipartEncoder
+
+from fiddler3.constants.common import MULTI_PART_CHUNK_SIZE
+from fiddler3.decorators import handle_api_error
+from fiddler3.entities.base import BaseEntity
+from fiddler3.entities.user import CreatedByMixin, UpdatedByMixin
+from fiddler3.schemas.file import FileResp
+from fiddler3.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class File(BaseEntity, CreatedByMixin, UpdatedByMixin):
+    def __init__(self, path: str | Path, name: str | None = None) -> None:
+        """Construct a files instance."""
+        self.path = path
+        self.name = name or os.path.basename(self.path)
+        self.id: UUID | None = None
+        self.status: str | None = None
+        self.type: str | None = None
+
+        self.created_at: datetime | None = None
+        self.updated_at: datetime | None = None
+
+        # Deserialized response object
+        self._resp: FileResp | None = None
+
+    @classmethod
+    def _from_dict(cls, data: dict) -> File:
+        """Build entity object from the given dictionary"""
+        raise NotImplementedError()
+
+    def _refresh(self, data: dict) -> None:
+        """Refresh the fields of this instance from the given response dictionary"""
+        resp_obj = FileResp(**data)
+
+        # Reset fields
+        self.name = resp_obj.name
+
+        # Add remaining fields
+        fields = [
+            'id',
+            'status',
+            'type',
+            'created_at',
+            'updated_at',
+        ]
+        for field in fields:
+            setattr(self, field, getattr(resp_obj, field, None))
+
+        self._resp = resp_obj
+
+    @handle_api_error
+    def upload(
+        self,
+        chunk_size: int = MULTI_PART_CHUNK_SIZE,
+    ) -> File:
+        """Upload file. Single or multipart upload depends on file size"""
+        if os.path.getsize(self.path) < chunk_size:
+            return self.single_upload()
+
+        return self.multipart_upload(chunk_size=chunk_size)
+
+    @handle_api_error
+    def single_upload(self) -> File:
+        """Single part file upload"""
+        with open(self.path, 'rb') as f:
+            multipart_data = MultipartEncoder(fields={'file': (self.name, f)})
+
+            response = self._client().post(
+                url='/v3/files/upload',
+                data=multipart_data,
+                headers={'Content-Type': multipart_data.content_type},
+            )
+
+        self._refresh_from_response(response)
+        return self
+
+    @handle_api_error
+    def multipart_upload(
+        self,
+        chunk_size: int = MULTI_PART_CHUNK_SIZE,
+    ) -> File:
+        """Multi part file upload"""
+
+        # init multipart upload
+        self._init_multipart_upload()
+
+        assert self.id is not None
+        # multipart parts upload
+        parts = self._multipart_upload(
+            chunk_size=chunk_size,
+        )
+
+        # multipart upload complete
+        response = self._multipart_upload_complete(parts=parts)
+
+        self._refresh_from_response(response)
+        return self
+
+    def _init_multipart_upload(self) -> File:
+        logger.info('Multi-part upload initializing')
+        response = self._client().post(
+            url='/v3/files/multipart-init',
+            data={'filename': self.name},
+        )
+
+        self._refresh_from_response(response)
+        return self
+
+    def _multipart_upload(self, chunk_size: int) -> list[dict[str, Any]]:
+        parts: list[dict[str, Any]] = []
+        part_number = 1
+
+        with open(self.path, 'rb') as f:
+            while True:
+                data = f.read(chunk_size)
+                if not data:
+                    break
+
+                logger.info('Uploading part %d', part_number)
+
+                response = self._client().post(
+                    url='/v3/files/multipart-upload',
+                    files={
+                        'file': (f'part_{part_number}_{self.name}', io.BytesIO(data)),
+                        'file_id': (None, str(self.id)),
+                        'part_number': (None, part_number),
+                    },
+                    headers={'Content-Type': None},
+                )
+
+                parts.append(
+                    {'number': part_number, 'id': response.json()['data']['part_id']}
+                )
+
+                part_number += 1
+
+        return parts
+
+    def _multipart_upload_complete(self, parts: list[dict[str, Any]]) -> Response:
+
+        response = self._client().post(
+            url='/v3/files/multipart-complete',
+            json={
+                'file_id': str(self.id),
+                'parts': parts,
+            },
+        )
+
+        logger.info('Multi-part upload complete')
+        return response

fiddler3/entities/model.py

@@ -4,29 +4,34 @@ import builtins
 import typing
 from dataclasses import dataclass
 from datetime import datetime
+from functools import cached_property
 from pathlib import Path
 from typing import Any, Iterator
 from uuid import UUID
 
 import pandas as pd
 
+from fiddler3.constants.dataset import EnvType
 from fiddler3.constants.model import ModelInputType, ModelTask
 from fiddler3.decorators import handle_api_error
 from fiddler3.entities.base import BaseEntity
-from fiddler3.entities.helpers import raise_not_found
+from fiddler3.entities.events import EventPublisher
 from fiddler3.entities.job import Job
-from fiddler3.entities.model_artifact import ArtifactMixin
-from fiddler3.entities.model_surrogate import SurrogateMixin
+from fiddler3.entities.model_artifact import ModelArtifact
+from fiddler3.entities.model_deployment import ModelDeployment
 from fiddler3.entities.project import ProjectCompactMixin
+from fiddler3.entities.surrogate import Surrogate
 from fiddler3.entities.user import CreatedByMixin, UpdatedByMixin
 from fiddler3.entities.xai import XaiMixin
+from fiddler3.schemas.deployment_params import DeploymentParams
 from fiddler3.schemas.filter_query import OperatorType, QueryCondition, QueryRule
 from fiddler3.schemas.job import JobCompactResp
 from fiddler3.schemas.model import ModelResp
-from fiddler3.schemas.model_schema import Column, ModelSchema
+from fiddler3.schemas.model_schema import ModelSchema
 from fiddler3.schemas.model_spec import ModelSpec
 from fiddler3.schemas.model_task_params import ModelTaskParams
 from fiddler3.schemas.xai_params import XaiParams
+from fiddler3.utils.helpers import raise_not_found
 from fiddler3.utils.model_schema_generator import SchemaGeneratorFactory
 
 if typing.TYPE_CHECKING:
@@ -35,11 +40,9 @@ if typing.TYPE_CHECKING:
 
 class Model(
     BaseEntity,
-    ArtifactMixin,
     CreatedByMixin,
     ProjectCompactMixin,
     UpdatedByMixin,
-    SurrogateMixin,
     XaiMixin,
 ):  # pylint: disable=too-many-ancestors
     def __init__(  # pylint: disable=too-many-arguments
@@ -47,7 +50,7 @@ class Model(
         name: str,
         project_id: UUID,
         schema: ModelSchema,
-        spec: ModelSpec | None = None,
+        spec: ModelSpec,
         input_type: str = ModelInputType.TABULAR,
         task: str = ModelTask.NOT_SET,
         task_params: ModelTaskParams | None = None,
@@ -67,18 +70,13 @@ class Model(
         self.event_id_col = event_id_col
         self.event_ts_col = event_ts_col
         self.event_ts_format = event_ts_format
-        self.spec = spec or ModelSpec()
+        self.spec = spec
         self.task_params = task_params or ModelTaskParams()
         self.xai_params = xai_params or XaiParams()
 
         self.id: UUID | None = None
         self.artifact_status: str | None = None
         self.artifact_files: list[dict] | None = None
-        self.input_cols: list[Column] | None = None
-        self.output_cols: list[Column] | None = None
-        self.target_cols: list[Column] | None = None
-        self.metadata_cols: list[Column] | None = None
-        self.decision_cols: list[Column] | None = None
         self.is_binary_ranking_model: bool | None = None
         self.created_at: datetime | None = None
         self.updated_at: datetime | None = None
@@ -122,11 +120,6 @@ class Model(
             'updated_at',
             'artifact_status',
             'artifact_files',
-            'input_cols',
-            'output_cols',
-            'target_cols',
-            'metadata_cols',
-            'decision_cols',
             'is_binary_ranking_model',
         ]
         for field in fields:
@@ -160,11 +153,6 @@ class Model(
             'updated_at',
             'artifact_status',
             'artifact_files',
-            'input_cols',
-            'output_cols',
-            'target_cols',
-            'metadata_cols',
-            'decision_cols',
             'is_binary_ranking_model',
         ]
         for field in fields:
@@ -172,6 +160,24 @@ class Model(
 
         self._resp = resp_obj
 
+    @cached_property
+    def _artifact(self) -> ModelArtifact:
+        """Model artifact instance"""
+        assert self.id is not None
+        return ModelArtifact(model_id=self.id)
+
+    @cached_property
+    def _surrogate(self) -> Surrogate:
+        """Model artifact instance"""
+        assert self.id is not None
+        return Surrogate(model_id=self.id)
+
+    @cached_property
+    def _event_publisher(self) -> EventPublisher:
+        """Event publisher instance"""
+        assert self.id is not None
+        return EventPublisher(model_id=self.id)
+
     @classmethod
     @handle_api_error
     def get(cls, id_: UUID | str) -> Model:
@@ -201,7 +207,7 @@ class Model(
         if response.json()['data']['total'] == 0:
             raise_not_found('Model not found for the given identifier')
 
-        return cls._from_dict(data=response.json()['data']['items'][0])
+        return cls.get(id_=response.json()['data']['items'][0]['id'])
 
     @handle_api_error
     def create(self) -> Model:
@@ -268,6 +274,14 @@ class Model(
 
         yield from Dataset.list(model_id=self.id)
 
+    @property
+    @handle_api_error
+    def model_deployment(self) -> ModelDeployment:
+        """Fetch all the deployments for this model"""
+        assert self.id is not None
+
+        return ModelDeployment.of(model_id=self.id)
+
     @classmethod
     @handle_api_error
     def generate_schema(
@@ -304,6 +318,109 @@ class Model(
         job_compact = JobCompactResp(**response.json()['data']['job'])
         return Job.get(id_=job_compact.id)
 
+    @handle_api_error
+    def publish(
+        self,
+        source: builtins.list[dict[str, Any]] | str | Path | pd.DataFrame,
+        environment: EnvType = EnvType.PRODUCTION,
+        dataset_name: str | None = None,
+        update: bool = False,
+    ) -> builtins.list[UUID] | Job:
+        """
+        Publish Pre-production or Production data
+
+        :param source: one of:
+            Path or str path: path for data file.
+            list[dict]: list of events (max 1000) PRE_PRODUCTION environment is not
+                supported
+            dataframe: events dataframe. EnvType.PRE_PRODUCTION not supported.
+        :param environment: Either EnvType.PRE_PRODUCTION or EnvType.PRODUCTION
+        :param dataset_name: Name of the dataset. Not supported for EnvType.PRODUCTION
+        :param update: flag indicating if the events are updates to previously
+            published rows
+
+        :return: list[UUID] for list of dicts or dataframe source and Job object for
+            file path source.
+        """
+
+        return self._event_publisher.publish(
+            source=source,
+            environment=environment,
+            dataset_name=dataset_name,
+            update=update,
+        )
+
+    def add_artifact(
+        self,
+        model_dir: str | Path,
+        deployment_params: DeploymentParams | None = None,
+    ) -> Job:
+        """
+        Upload and deploy model artifact.
+
+        :param model_dir: Path to model artifact tar file
+        :param deployment_params: Model deployment parameters
+        :return: Async job instance
+        """
+        return self._artifact.add(
+            model_dir=model_dir, deployment_params=deployment_params
+        )
+
+    def update_artifact(
+        self,
+        model_dir: str | Path,
+        deployment_params: DeploymentParams | None = None,
+    ) -> Job:
+        """
+        Update existing model artifact.
+
+        :param model_dir: Path to model artifact tar file
+        :param deployment_params: Model deployment parameters
+        :return: Async job instance
+        """
+        return self._artifact.update(
+            model_dir=model_dir, deployment_params=deployment_params
+        )
+
+    def download_artifact(
+        self,
+        output_dir: str | Path,
+    ) -> None:
+        """
+        Download existing model artifact.
+
+        :param output_dir: Path to download model artifact tar file
+        """
+
+        self._artifact.download(output_dir=output_dir)
+
+    def add_surrogate(
+        self,
+        deployment_params: DeploymentParams | None = None,
+    ) -> Job:
+        """
+        Add a new surrogate model
+
+        :param deployment_params: Model deployment parameters
+        :return: Async job
+        """
+        job = self._surrogate.add(deployment_params=deployment_params)
+        return job
+
+    @handle_api_error
+    def update_surrogate(
+        self,
+        deployment_params: DeploymentParams | None = None,
+    ) -> Job:
+        """
+        Update an existing surrogate model
+
+        :param deployment_params: Model deployment parameters
+        :return: Async job
+        """
+        job = self._surrogate.update(deployment_params=deployment_params)
+        return job
+
 
 @dataclass
 class ModelCompact: