wizata-dsapi 1.3.39__tar.gz → 1.3.41__tar.gz

{wizata_dsapi-1.3.39/wizata_dsapi.egg-info → wizata_dsapi-1.3.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 1.3.39
+Version: 1.3.41
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-1.3.39 → wizata_dsapi-1.3.41}/wizata_dsapi/api_dto.py

@@ -2,6 +2,12 @@ import uuid
 from enum import Enum
 
 
+class ApiDtoInterface:
+
+    def load_model(self, model):
+        pass
+
+
 class VarType(Enum):
     """
     defines possible type for a defined variable.

{wizata_dsapi-1.3.39 → wizata_dsapi-1.3.41}/wizata_dsapi/mlmodel.py

@@ -1,6 +1,6 @@
 from typing import List, Iterator, Union, Optional
 import os
-from .api_dto import ApiDto
+from .api_dto import ApiDto, ApiDtoInterface
 from datetime import datetime, timezone
 
 
@@ -132,7 +132,8 @@ class ModelInfo:
                  property_name: str = None,
                  trained_model = None,
                  scaler = None,
-                 files = None
+                 files = None,
+                 updated_date = None
                  ):
         # information identifying a model
         self.key = key
@@ -152,6 +153,7 @@ class ModelInfo:
                 if not isinstance(file, ModelFile):
                     raise TypeError("file is not a ModelFile with files")
         self.is_active = False
+        self.updated_date = None
 
         # files attached to model when loaded
         self.trained_model = trained_model
@@ -163,6 +165,17 @@ class ModelInfo:
         self.has_target_feat = False
         self.label_counts = 0
 
+        # api
+        self._api = None
+
+    def bind_api(self, api:ApiDtoInterface):
+        """
+        internal method to bind the api to the dto.
+        :param api: api client
+        :return: None
+        """
+        self._api = api
+
     @classmethod
     def split_identifier(cls, identifier: str):
         """
@@ -249,6 +262,8 @@ class ModelInfo:
             obj["identifier"] = self.identifier(include_alias=True)
         if len(self.files) > 0:
             obj["files"] = [file.to_json() for file in self.files]
+        if self.updated_date is not None:
+            obj["updatedDate"] = str(self.updated_date)
         return obj
 
     def from_json(self, obj):
@@ -273,6 +288,21 @@ class ModelInfo:
             self.property_name = obj["property_name"]
         if "is_active" in obj.keys():
             self.is_active = get_bool(obj, name="is_active")
+        if "updatedDate" in obj.keys() and obj["updatedDate"] is not None:
+            self.updated_date = obj["updatedDate"]
+        if "files" in obj.keys():
+            for obj_file in obj["files"]:
+                model_file = ModelFile()
+                model_file.from_json(obj_file)
+                self.add_file(model_file)
+
+    def load(self):
+        """
+        load the trained model from the repository.
+        """
+        if self._api is None:
+            raise RuntimeError("api is not bound to the dto use bind_api()")
+        self._api.load_model(self)
 
 
 class ModelList:
@@ -298,15 +328,23 @@ class ModelList:
         """
         return any(model_in_list.identifier(include_alias=True) == model.identifier(include_alias=True) for model_in_list in self.models)
 
-    def __getitem__(self, key: Union[int, ModelInfo]) -> ModelInfo:
+    def __getitem__(self, key: Union[int, str, ModelInfo]) -> ModelInfo:
         """
-        find a model within list based on key or ModelInfo.
-        :param key: ModelInfo or index
+        find a model within list based on index, identifier or ModelInfo.
+        :param key: identifier or ModelInfo or index
         :return: the model_info
         """
         if isinstance(key, int):
             return self.models[key]
 
+        elif isinstance(key, str):
+            if "@" not in key:
+                return self.select_active_model(identifier=key)
+            for model in self.models:
+                if model.identifier(include_alias=True) == key:
+                    return model
+            raise KeyError(f"model with identifier '{key}' not found within this ModelList.")
+
         elif isinstance(key, ModelInfo):
             identifier = key.identifier(include_alias=True)
             for model in self.models:
@@ -315,7 +353,22 @@ class ModelList:
             raise KeyError(f"model with identifier '{identifier}' not found within this ModelList.")
 
         else:
-            raise TypeError("ModelList indices must be int or ModelInfo.")
+            raise TypeError("ModelList indices must be int, str or ModelInfo.")
+
+    def select_active_model(self, identifier: str) -> ModelInfo:
+        """
+        return the active model based on active status or latest one if none active.
+        :param identifier: identifier
+        :return: active model
+        """
+        models = []
+        for model in self.models:
+            if model.identifier(include_alias=False) == identifier:
+                if model.is_active:
+                    return model
+                else:
+                    models.append(model)
+        return max(models, key=lambda f: f.updated_date, default=None)
 
     def append(self, model: ModelInfo):
         self.models.append(model)

{wizata_dsapi-1.3.39 → wizata_dsapi-1.3.41}/wizata_dsapi/pipeline.py

@@ -1,5 +1,6 @@
 import uuid
 import json
+import sys
 
 import pandas
 import wizata_dsapi
@@ -159,13 +160,13 @@ class PipelineIO(ApiDto):
             obj["columns"] = self.columns
         return obj
 
-    def prepare(self, df: pandas.DataFrame) -> pandas.DataFrame:
+    def _prepare_df(self, df: pandas.DataFrame) -> pandas.DataFrame:
         """
-        prepare the dataframe based in information from this pipeline I/O definition.
-        perform mapping, selection and drops.
+        prepare dataframe in both 3.9 and 3.11+
         :param pandas.DataFrame df: dataframe to prepare.
         :return: prepared dataframe.
         """
+
         try:
             prepare_df = df.copy()
 
@@ -186,6 +187,25 @@ class PipelineIO(ApiDto):
         except Exception as e:
             raise RuntimeError(f'not able to prepare your dataframe following Pipeline I/O directives {e}')
 
+    if sys.version_info >= (3, 11):
+
+        ## with torch support
+        import torch
+        from typing import Union
+
+        def prepare(self, df: Union[pandas.DataFrame, torch.Tensor]) -> Union[pandas.DataFrame, torch.Tensor]:
+            import torch
+            if isinstance(df, torch.Tensor):
+                return df
+            else:
+                return self._prepare_df(df)
+
+    else:
+
+        ## without torch support
+        def prepare(self, df: pandas.DataFrame) -> pandas.DataFrame:
+            return self._prepare_df(df)
+
     @classmethod
     def from_obj(cls, obj):
         """

wizata_dsapi-1.3.41/wizata_dsapi/version.py

@@ -0,0 +1 @@
+__version__ = "1.3.41"

{wizata_dsapi-1.3.39 → wizata_dsapi-1.3.41}/wizata_dsapi/wizata_dsapi_client.py

@@ -16,11 +16,13 @@ import types
 import wizata_dsapi
 import urllib.parse
 import base64
+import joblib
+import io
 
 import string
 import random
 
-from .api_dto import ApiDto, VarType
+from .api_dto import ApiDto, VarType, ApiDtoInterface
 from .business_label import BusinessLabel
 from .plot import Plot
 from .request import Request
@@ -69,7 +71,7 @@ def parse_string_list(s):
         return []
 
 
-class WizataDSAPIClient(ApiInterface):
+class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
     """
     client wrapper to cloud data science API
 
@@ -1100,17 +1102,24 @@ class WizataDSAPIClient(ApiInterface):
             raise self.__raise_error(response)
 
     def upload_model(self,
-                     model_info: ModelInfo):
+                     model_info: ModelInfo,
+                     bytes_content = None):
         """
         upload a model within the model repository.
+            - by default use model_info.trained_model and convert it to a pickle
+            - for already torch or pickle please pass the bytes_content
+            - model_info.file_format must be set properly to 'pkl' or 'pt'
         :param model_info: model info, with at least key (+twin, +property, +alias) and trained_model.
+        :param bytes_content: bytes[] of your torch or pickle model.
         """
-        if model_info.trained_model is None:
-            raise ValueError("model_info must have a trained model as bytes content pkl or pt")
+        if model_info.trained_model is None and bytes_content is None:
+            raise ValueError("model_info must have a trained model (to pickle) or bytes content")
+        if bytes_content is None:
+            bytes_content = pickle.dumps(model_info.trained_model)
         files = {
             "trained_model": (
                 "trained_model." + model_info.file_format ,
-                model_info.trained_model,
+                bytes_content,
                 "application/octet-stream",
             )
         }
@@ -1849,6 +1858,27 @@ class WizataDSAPIClient(ApiInterface):
         else:
             raise self.__raise_error(response)
 
+    def search_models(self) -> ModelList:
+        """
+        get all information related to models stored on Wizata.
+        :return: ModelList structure model list
+        """
+        response = requests.request("GET",
+                                    self.__url() + "models",
+                                    headers=self.__header()
+                                    )
+        if response.status_code == 200:
+            response_json = response.json()
+            model_list = ModelList()
+            for model_json in response_json:
+                model_info = ModelInfo(model_json["key"])
+                model_info.from_json(model_json)
+                model_info._api = self
+                model_list.append(model_info)
+            return model_list
+        else:
+            raise self.__raise_error(response)
+
     def abort(self, executions: list) -> str:
         """
         send an abort request for executions and return a result message
@@ -1905,6 +1935,32 @@ class WizataDSAPIClient(ApiInterface):
         image = PipelineImage.loads(pipeline_image_id=pipeline_image_id, g_bytes=response_bytes)
         return image
 
+    def load_model(self, model):
+        """
+        load a model pickle or torch from the repository ready to be used.
+        :param model: ModelInfo to load
+        :return: ModelInfo with the trained model.
+        """
+        if not isinstance(model, ModelInfo):
+            raise TypeError('model must be an instance of ModelInfo')
+
+        identifier = model.identifier(include_alias=True)
+        extension = model.file_format
+        response = requests.get(self.__url() + f"models/{identifier}/files/trained_model.{extension}/",
+                                headers=self.__header())
+        if response.status_code == 200:
+            if extension == 'pkl':
+                model.trained_model = joblib.load(io.BytesIO(response.content))
+                return model
+            elif extension == 'pt':
+                import torch
+                model.trained_model = torch.jit.load(io.BytesIO(response.content))
+                return model
+            else:
+                raise ValueError(f'unsupported file format {extension}')
+        else:
+            self.__raise_error(response)
+
 
 def api() -> WizataDSAPIClient:
     """

{wizata_dsapi-1.3.39 → wizata_dsapi-1.3.41/wizata_dsapi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 1.3.39
+Version: 1.3.41
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

wizata_dsapi-1.3.39/wizata_dsapi/version.py

@@ -1 +0,0 @@
-__version__ = "1.3.39"