ml-management 0.4.1__tar.gz → 0.5.0__tar.gz

ml_management-0.5.0/ML_management/__init__.py

@@ -0,0 +1,3 @@
+import ML_management.version
+
+__version__ = ML_management.version.__version__

{ml-management-0.4.1/ML_management/mlmanagement → ml_management-0.5.0/ML_management}/base_exceptions.py

@@ -1,6 +1,3 @@
-"""Base classes for custom exceptions used in project."""
-
-
 class MLMBaseError(Exception):
     """Base exception for all custom exceptions."""
 
@@ -23,10 +20,3 @@ class MLMClientError(MLMBaseError):
     """Base exception for all client-side specific exceptions."""
 
     pass
-
-
-class PylintError(MLMBaseError):
-    """Base exception for all errors within linter check."""
-
-    def __init__(self, message):
-        super().__init__("Pylint found errors in code of uploaded modules:\n" + message)

{ml-management-0.4.1 → ml_management-0.5.0}/ML_management/collectors/s3/s3collector.py

@@ -16,8 +16,8 @@ class S3Collector(CollectorPattern):
             "type": "object",
             "properties": {
                 "bucket": {"type": "string"},
-                "untar_data": {"type": "boolean"},
-                "remote_paths": {"type": "array", "items": {"type": "string"}},
+                "untar_data": {"type": "boolean", "default": False},
+                "remote_paths": {"type": ["array", None], "items": {"type": "string"}, "default": None},
             },
             "required": ["bucket"],
             "additionalProperties": False,

ml_management-0.5.0/ML_management/dataset_loader/dataset_loader_pattern.py

@@ -0,0 +1,32 @@
+"""Dataset loader template for custom dataset loader."""
+from abc import ABC, abstractmethod
+
+
+class DatasetLoaderPattern(ABC):
+    """Define dataset loader.
+
+    Attributes
+    ----------
+    artifacts : str
+        Local path to artifacts.
+        That parameters will be set automatically in job before the 'get_dataset' func would be executed.
+    data_path: str
+        A path to data to be loaded.
+
+    """
+
+    def __init__(self):
+        """Init dataset loader class."""
+        self.artifacts: str
+        self.data_path = None
+
+    @abstractmethod
+    def get_dataset(self, **dataset_params):
+        """
+        Return dataset.
+
+        To get data_path use self.data_path parameter, which also will be set in the job.
+        'dataset_params' are dataset_loader parameters. One has to define it as ordinary kwargs
+        with type annotation.
+        """
+        raise NotImplementedError

{ml-management-0.4.1 → ml_management-0.5.0}/ML_management/executor/base_executor.py

@@ -1,10 +1,8 @@
 """Executor template for custom executor."""
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Union
 
-import ML_management.mlmanagement.log_api
-import ML_management.mlmanagement.mlmanager
 from ML_management.dataset_loader import DatasetLoaderPattern
 from ML_management.dataset_loader.dataset_loader_pattern_to_methods_map import DatasetLoaderMethodName
 from ML_management.executor.patterns import (
@@ -14,17 +12,13 @@ from ML_management.executor.patterns import (
     OneDatasetLoaderPattern,
     OneModelPattern,
 )
-from ML_management.mlmanagement import variables
-from ML_management.mlmanagement.module_finder import ModuleFinder
-from ML_management.mlmanagement.visibility_options import VisibilityOptions
 from ML_management.model.model_type_to_methods_map import ModelMethodName
 from ML_management.model.patterns.model_pattern import Model
-from ML_management.model.patterns.rich_python_model import RichPythonModel
 
 default_dataset_loader_pattern = OneDatasetLoaderPattern()
 
 
-class BaseExecutor(RichPythonModel, ABC):
+class BaseExecutor(ABC):
     """Define custom job executor."""
 
     DEFAULT_ROLE = DEFAULT_ROLE
@@ -140,91 +134,6 @@ class BaseExecutor(RichPythonModel, ABC):
         """
         raise NotImplementedError
 
-    def upload_executor(
-        self,
-        executor_name: str,
-        pip_requirements=None,
-        description: Optional[str] = None,
-        extra_pip_requirements=None,
-        conda_env=None,
-        artifacts: Optional[dict] = None,
-        visibility: VisibilityOptions = VisibilityOptions.PRIVATE,
-        extra_modules_names: Optional[List[str]] = None,
-        used_modules_names: Optional[List[str]] = None,
-        linter_check: bool = True,
-        start_build: bool = False,
-    ) -> None:
-        """
-        Upload wrapper to MLmanagement server.
-
-        :param pip_requirements: {{ pip_requirements }}
-
-        :param extra_pip_requirements: {{ extra_pip_requirements }}
-        `pip_requirements` and 'extra_pip_requirements' must be either a string path to a pip requirements file on the
-            local filesystem or an iterable of pip requirement strings.
-
-        :param conda_env: {{ conda_env }}
-        'conda_env' must be a dict specifying the conda environment for this model.
-
-        :param artifacts: A dictionary containing ``<name, artifact_uri>`` entries. Remote artifact URIs
-                          are resolved to absolute filesystem paths, producing a dictionary of
-                          ``<name, absolute_path>`` entries. ``python_model`` can reference these
-                          resolved entries as the ``artifacts`` property of the ``context`` parameter
-                          in :func:`PythonModel.load_context() <mlflow.pyfunc.PythonModel.load_context>`
-                          and :func:`PythonModel.predict() <mlflow.pyfunc.PythonModel.predict>`.
-                          For example, consider the following ``artifacts`` dictionary::
-
-                            {
-                                "my_file": "s3://my-bucket/path/to/my/file"
-                            }
-
-                          In this case, the ``"my_file"`` artifact is downloaded from S3. The
-                          ``python_model`` can then refer to ``"my_file"`` as an absolute filesystem
-                          path via ``context.artifacts["my_file"]``.
-
-                          If ``None``, no artifacts are added to the executor.
-
-        :param visibility: either a private or public executor.
-
-        :param extra_modules_names: names of modules that should be pickled by value
-            in addition to auto-detected modules.
-
-        :param used_modules_names: modules that should be pickled by value, disables the auto-detection of modules.
-
-        :param linter_check: if True, check code of the executor by linter.
-
-        :param start_build: if set to True, start job to create docker image.
-            This parameter may be True only if there is no models for executor.
-        """
-        old_experiment_name = variables.active_experiment
-        if (self.desired_model_methods or self.upload_model_modes) and start_build:
-            raise RuntimeError("Parameter start_build=True is acceptable only if there is no models for executor.")
-        ML_management.mlmanagement.mlmanager.set_experiment(
-            variables.EXPERIMENT_NAME_FOR_EXECUTOR, visibility=VisibilityOptions.PUBLIC
-        )
-        try:
-            with ML_management.mlmanagement.mlmanager.start_run(nested=True):
-                ML_management.mlmanagement.log_api._log_model(
-                    artifact_path="",
-                    description=description,
-                    artifacts=artifacts,
-                    python_model=self,
-                    registered_model_name=executor_name,
-                    pip_requirements=pip_requirements,
-                    extra_pip_requirements=extra_pip_requirements,
-                    conda_env=conda_env,
-                    visibility=visibility,
-                    extra_modules_names=extra_modules_names,
-                    used_modules_names=used_modules_names,
-                    root_module_name=ModuleFinder.get_my_caller_module_name(),
-                    linter_check=linter_check,
-                    start_build=start_build,
-                )
-        except Exception as err:
-            raise err
-        finally:
-            variables.active_experiment = old_experiment_name
-
     @property
     def model(self) -> Model:
         """Property returning a single model.