ml-management 0.6.3rc15__tar.gz → 0.6.3rc16__tar.gz

ml_management-0.6.3rc16/ML_management/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.3rc16"

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/ML_management/graphql/schema.py

@@ -1128,7 +1128,7 @@ class JobPagination(sgqlc.types.Type):
 
 class JobParams(sgqlc.types.Type):
     __schema__ = schema
-    __field_names__ = ('additional_system_packages', 'cron_expression', 'executor_params', 'experiment', 'experiment_id', 'gpu', 'list_role_data_params', 'list_role_model_params', 'periodic_type')
+    __field_names__ = ('additional_system_packages', 'cron_expression', 'executor_params', 'experiment', 'experiment_id', 'gpu', 'list_role_data_params', 'list_role_model_params', 'models_schemas', 'periodic_type')
     additional_system_packages = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name='additionalSystemPackages')
 
     cron_expression = sgqlc.types.Field(String, graphql_name='cronExpression')
@@ -1145,6 +1145,8 @@ class JobParams(sgqlc.types.Type):
 
     list_role_model_params = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null('RoleModelParams')), graphql_name='listRoleModelParams')
 
+    models_schemas = sgqlc.types.Field(sgqlc.types.non_null(JSON), graphql_name='modelsSchemas')
+
     periodic_type = sgqlc.types.Field(PeriodicType, graphql_name='periodicType')
 
 
@@ -1252,7 +1254,7 @@ class ModelPagination(sgqlc.types.Type):
 
 class ModelParams(sgqlc.types.Type):
     __schema__ = schema
-    __field_names__ = ('description', 'list_model_method_params', 'model_version_choice', 'new_model_name', 'prepare_new_model_inference')
+    __field_names__ = ('description', 'list_model_method_params', 'model_version_choice', 'new_model_name', 'new_model_visibility', 'prepare_new_model_inference', 'start_build_new_model_image')
     description = sgqlc.types.Field(String, graphql_name='description')
 
     list_model_method_params = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MethodParams))), graphql_name='listModelMethodParams')
@@ -1261,8 +1263,12 @@ class ModelParams(sgqlc.types.Type):
 
     new_model_name = sgqlc.types.Field(String, graphql_name='newModelName')
 
+    new_model_visibility = sgqlc.types.Field(sgqlc.types.non_null(VisibilityOptions), graphql_name='newModelVisibility')
+
     prepare_new_model_inference = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name='prepareNewModelInference')
 
+    start_build_new_model_image = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name='startBuildNewModelImage')
+
 
 
 class ModelVersionChoiceParams(sgqlc.types.Type):
@@ -1562,7 +1568,7 @@ class ModelVersionPagination(sgqlc.types.Type):
 
 class Mutation(sgqlc.types.Type):
     __schema__ = schema
-    __field_names__ = ('add_ml_job', 'cancel_build_job_for_executor_version', 'cancel_build_job_for_model_version', 'cancel_job', 'cancel_venv_build_job_for_model_version', 'create_experiment', 'create_local_job', 'delete_dataset_loader', 'delete_dataset_loader_tag', 'delete_dataset_loader_version_from_name_version', 'delete_dataset_loader_version_tag', 'delete_executor', 'delete_executor_tag', 'delete_executor_version_from_name_version', 'delete_executor_version_tag', 'delete_experiment_tag', 'delete_model', 'delete_model_tag', 'delete_model_version_from_name_version', 'delete_model_version_tag', 'log_metric', 'log_metrics', 'log_param', 'log_params', 'rebuild_model_version_image', 'rebuild_no_model_executor_version_image', 'rename_experiment', 'serve_model', 'set_dataset_loader_tag', 'set_dataset_loader_version_tag', 'set_executor_tag', 'set_executor_version_tag', 'set_experiment_tag', 'set_model_tag', 'set_model_version_tag', 'start_local_job', 'stop_local_job', 'stop_model_serving', 'update_dataset_loader', 'update_dataset_loader_version', 'update_executor', 'update_executor_version', 'update_experiment', 'update_job', 'update_model', 'update_model_version')
+    __field_names__ = ('add_ml_job', 'cancel_build_job_for_executor_version', 'cancel_build_job_for_model_version', 'cancel_job', 'cancel_venv_build_job_for_model_version', 'create_experiment', 'create_local_job', 'delete_dataset_loader', 'delete_dataset_loader_tag', 'delete_dataset_loader_version_from_name_version', 'delete_dataset_loader_version_tag', 'delete_executor', 'delete_executor_tag', 'delete_executor_version_from_name_version', 'delete_executor_version_tag', 'delete_experiment_tag', 'delete_model', 'delete_model_tag', 'delete_model_version_from_name_version', 'delete_model_version_tag', 'log_metric', 'log_metrics', 'log_param', 'log_params', 'rebuild_model_version_image', 'rebuild_no_model_executor_version_image', 'rename_experiment', 'serve_model', 'set_dataset_loader_tag', 'set_dataset_loader_version_tag', 'set_executor_tag', 'set_executor_version_tag', 'set_experiment_tag', 'set_model_tag', 'set_model_version_tag', 'start_job', 'stop_job', 'stop_model_serving', 'update_dataset_loader', 'update_dataset_loader_version', 'update_executor', 'update_executor_version', 'update_experiment', 'update_job', 'update_model', 'update_model_version')
     add_ml_job = sgqlc.types.Field(sgqlc.types.non_null(ExecutionJob), graphql_name='addMlJob', args=sgqlc.types.ArgDict((
         ('form', sgqlc.types.Arg(sgqlc.types.non_null(JobParameters), graphql_name='form', default=None)),
 ))
@@ -1627,7 +1633,7 @@ class Mutation(sgqlc.types.Type):
     * `visibility` (`VisibilityOptions!`)None (default: `PRIVATE`)
     '''
 
-    create_local_job = sgqlc.types.Field(sgqlc.types.non_null(ExecutionJob), graphql_name='createLocalJob', args=sgqlc.types.ArgDict((
+    create_local_job = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name='createLocalJob', args=sgqlc.types.ArgDict((
         ('experiment_name', sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name='experimentName', default=None)),
         ('job_name', sgqlc.types.Arg(String, graphql_name='jobName', default=None)),
         ('visibility', sgqlc.types.Arg(sgqlc.types.non_null(VisibilityOptions), graphql_name='visibility', default=None)),
@@ -1944,16 +1950,17 @@ class Mutation(sgqlc.types.Type):
     * `value` (`String!`)None
     '''
 
-    start_local_job = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name='startLocalJob', args=sgqlc.types.ArgDict((
-        ('job_name', sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name='jobName', default=None)),
+    start_job = sgqlc.types.Field(sgqlc.types.non_null(ExecutionJob), graphql_name='startJob', args=sgqlc.types.ArgDict((
+        ('secret_uuid', sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name='secretUuid', default=None)),
 ))
     )
     '''Arguments:
 
-    * `job_name` (`String!`)None
+    * `secret_uuid` (`String!`)None
     '''
 
-    stop_local_job = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name='stopLocalJob', args=sgqlc.types.ArgDict((
+    stop_job = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name='stopJob', args=sgqlc.types.ArgDict((
+        ('exception_traceback', sgqlc.types.Arg(String, graphql_name='exceptionTraceback', default=None)),
         ('message', sgqlc.types.Arg(String, graphql_name='message', default=None)),
         ('secret_uuid', sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name='secretUuid', default=None)),
         ('status', sgqlc.types.Arg(sgqlc.types.non_null(JobStatus), graphql_name='status', default='SUCCESSFUL')),
@@ -1961,6 +1968,7 @@ class Mutation(sgqlc.types.Type):
     )
     '''Arguments:
 
+    * `exception_traceback` (`String`)None (default: `null`)
     * `message` (`String`)None (default: `null`)
     * `secret_uuid` (`String!`)None
     * `status` (`JobStatus!`)None (default: `SUCCESSFUL`)

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/ML_management/mlmanagement/__init__.py

@@ -1,4 +1,4 @@
-from ML_management.mlmanagement import backend_api, load_api, log_api
+from ML_management.mlmanagement import active_job, backend_api, load_api, log_api
 
 log_executor_src = log_api.log_executor_src
 log_dataset_loader_src = log_api.log_dataset_loader_src
@@ -14,3 +14,4 @@ load_dataset = load_api.load_dataset
 load_executor = load_api.load_executor
 download_artifacts_by_name_version = load_api.download_artifacts_by_name_version
 download_job_artifacts = load_api.download_job_artifacts
+start_job = active_job.start_job

ml_management-0.6.3rc16/ML_management/mlmanagement/active_job.py

@@ -0,0 +1,99 @@
+import traceback
+from typing import Optional
+
+from sgqlc.operation import Operation
+
+from ML_management import variables
+from ML_management.graphql import schema
+from ML_management.graphql.send_graphql_request import send_graphql_request
+from ML_management.mlmanagement.batcher import Batcher
+from ML_management.mlmanagement.visibility_options import VisibilityOptions
+from ML_management.variables import DEFAULT_EXPERIMENT
+
+
+class ActiveJob:
+    """
+    A context manager that allows for the execution of a task locally.
+
+    This class provides a convenient way to run a job locally.
+
+    """
+
+    def __init__(self, secret_uuid):
+        self.secret_uuid = secret_uuid
+
+    def __enter__(self):
+        op = Operation(schema.Mutation)
+        base_query = op.start_job(secret_uuid=self.secret_uuid)
+        base_query.name()
+        base_query.experiment.name()
+        _query_job_params(base_query)
+        self.job = send_graphql_request(op=op, json_response=False).start_job
+        variables.secret_uuid = self.secret_uuid
+        return self.job
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        Batcher().wait_log_metrics()
+        exception_traceback = None
+        message = None
+        status = "SUCCESSFUL"
+        if exc_type:
+            exception_traceback = traceback.format_exc()
+            message = str(exc_val)
+            status = "FAILED"
+
+        op = Operation(schema.Mutation)
+        op.stop_job(
+            secret_uuid=variables.secret_uuid, status=status, message=message, exception_traceback=exception_traceback
+        )
+        try:
+            _ = send_graphql_request(op=op, json_response=False).stop_job
+        finally:
+            variables.secret_uuid = None
+
+
+def start_job(
+    job_name: Optional[str], experiment_name: str = DEFAULT_EXPERIMENT, visibility=VisibilityOptions.PRIVATE
+) -> ActiveJob:
+    """
+    Create local job.
+
+    Parameters
+    ----------
+    job_name: str | None
+        Name of the new job. If not passed, it will be generated.
+    experiment_name: str = "Default"
+        Name of the experiment. Default: "Default"
+    visibility: VisibilityOptions
+        Visibility of this job to other users. Default: PRIVATE.
+
+    Returns
+    -------
+    ActiveJob
+        Active job.
+
+    Usage:
+        with start_local_job('my-beautiful-job') as job:
+            mlmanagement.log_metric(...)
+            mlmanagement.log_artifacts(...)
+    """
+    op = Operation(schema.Mutation)
+    op.create_local_job(job_name=job_name, experiment_name=experiment_name, visibility=visibility.name)
+    secret_uuid = send_graphql_request(op=op, json_response=False).create_local_job
+    return ActiveJob(secret_uuid)
+
+
+def _query_job_params(base_query):
+    base_query.params()
+    base_query.params.gpu()
+    base_query.params.models_schemas()
+    base_query.params.list_role_model_params()
+    base_query.params.list_role_data_params()
+    base_query.params.list_role_data_params.data_params()
+    base_query.params.list_role_data_params.role()
+    base_query.params.list_role_model_params.model_params()
+    base_query.params.list_role_model_params.role()
+    base_query.params.executor_params()
+    base_query.params.executor_params.executor_method_params()
+    base_query.params.executor_params.executor_version_choice()
+    base_query.executor_version.upload_model_modes()

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/ML_management/mlmanagement/batcher.py

@@ -39,6 +39,7 @@ class Batcher(metaclass=Singleton):
         if self.thread:
             self.stop_event.set()
             self.thread.join()
+            self.thread = None
 
     def log_batch_metric(self, stop_event):
         while not stop_event.is_set():

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/ML_management/mlmanagement/load_api.py

@@ -28,9 +28,31 @@ from ML_management.variables import (
 
 
 def download_artifacts_by_name_version(
-    name: str, version: Optional[int], model_type: ModelType, path: str, dst_path: Optional[str] = None
+    name: str,
+    version: Optional[int],
+    model_type: ModelType,
+    path: str,
+    dst_path: Optional[str] = None,
 ) -> str:
-    """Download an artifact by name and version to a local directory, and return a local path for it."""
+    """Download an artifact by name and version to a local directory, and return a local path for it.
+
+    Parameters
+    ==========
+    name: str
+        Name of the entity.
+    version: Optional[int] = None
+        Version of the entity. Default: None, "latest" version is used.
+    model_type: ModelType
+        Type of the entity. Possible values: ModelType.MODEL | ModelType.EXECUTOR | ModelType.DATASET_LOADER
+    path: str = ""
+        Specific path for artifacts download. Default: "", all artifacts will be downloaded.
+    dst_path: Optional[str]: None
+        Destination path. Default: None.
+    Returns
+    =======
+    str
+        Local path to the entity folder.
+    """
     url = get_log_service_url("download_artifacts_by_name_version")
     params = {
         "path": os.path.normpath(path) if path else path,
@@ -41,10 +63,24 @@ def download_artifacts_by_name_version(
     return _request_download_artifacts(url, params, dst_path)
 
 
-def download_job_artifacts(job_id: str, path: str = "", dst_path: Optional[str] = None) -> str:
-    """Download an artifact file or directory from a job to a local directory, and return a local path for it."""
+def download_job_artifacts(name: str, path: str = "", dst_path: Optional[str] = None) -> str:
+    """Download an artifact file or directory from a job to a local directory, and return a local path for it.
+
+    Parameters
+    ==========
+    name: str
+        Name of the job.
+    path: str = ""
+        Specific path for artifacts download. Default: "", all artifacts will be downloaded.
+    dst_path: Optional[str]: None
+        Destination path. Default: None.
+    Returns
+    =======
+    str
+        Local path to artifacts.
+    """
     url = get_log_service_url("download_job_artifacts")
-    params = {"path": os.path.normpath(path) if path else path, "job_name": job_id}
+    params = {"path": os.path.normpath(path) if path else path, "job_name": name}
     return _request_download_artifacts(url, params, dst_path)
 
 
@@ -92,7 +128,14 @@ def load_dataset(
     DatasetLoaderPattern
         The object of the dataset to use.
     """
-    return _load_model_type(name, version, ModelType.DATASET_LOADER, install_requirements, dst_path, kwargs_for_init)
+    return _load_model_type(
+        name,
+        version,
+        ModelType.DATASET_LOADER,
+        install_requirements,
+        dst_path,
+        kwargs_for_init,
+    )
 
 
 def _set_model_version_requirements(local_path) -> None:
@@ -107,7 +150,15 @@ def _set_model_version_requirements(local_path) -> None:
     try:
         if requirements:
             subprocess.check_call(
-                [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--default-timeout=100", *requirements]
+                [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "install",
+                    "--no-cache-dir",
+                    "--default-timeout=100",
+                    *requirements,
+                ]
             )
 
     except Exception:

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/ML_management/sdk/job.py

@@ -15,7 +15,7 @@ from ML_management.collectors.collector_pattern_to_methods_map import collector_
 from ML_management.graphql import schema
 from ML_management.graphql.schema import ExecutionJob, Param
 from ML_management.graphql.send_graphql_request import send_graphql_request
-from ML_management.mlmanagement import get_server_url
+from ML_management.mlmanagement.backend_api import get_server_url
 from ML_management.mlmanagement.log_api import _raise_error
 from ML_management.mlmanagement.visibility_options import VisibilityOptions
 from ML_management.sdk.model import get_latest_model_version
@@ -47,8 +47,10 @@ def job_by_name(name: str) -> ExecutionJob:
     base_query.registration_timestamp()
     base_query.start_timestamp()
     base_query.end_timestamp()
-    base_query.build_job()
+    base_query.build_job.status()
+    base_query.build_job.message()
     base_query.message()
+    base_query.experiment.name()
 
     job = send_graphql_request(op, json_response=False)
 

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ml-management
-Version: 0.6.3rc15
+Version: 0.6.3rc16
 Summary: Python implementation of model pattern, dataset
 Author-Email: ISPRAS MODIS <modis@ispras.ru>
 Maintainer: Maxim Ryndin

{ml_management-0.6.3rc15 → ml_management-0.6.3rc16}/pyproject.toml

@@ -25,7 +25,7 @@ readme = "README.md"
 maintainers = [
     { name = "Maxim Ryndin" },
 ]
-version = "0.6.3rc15"
+version = "0.6.3rc16"
 
 [build-system]
 requires = [

ml_management-0.6.3rc15/ML_management/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.6.3rc15"

ml_management-0.6.3rc15/ML_management/mlmanagement/active_job.py

@@ -1,74 +0,0 @@
-from sgqlc.operation import Operation
-
-from ML_management import variables
-from ML_management.graphql import schema
-from ML_management.graphql.schema import ExecutionJob
-from ML_management.graphql.send_graphql_request import send_graphql_request
-from ML_management.mlmanagement.batcher import Batcher
-from ML_management.mlmanagement.visibility_options import VisibilityOptions
-from ML_management.variables import DEFAULT_EXPERIMENT
-
-
-class ActiveJob(ExecutionJob):
-    """
-    A context manager that allows for the execution of a task locally.
-
-    This class provides a convenient way to run a job locally.
-
-    """
-
-    def __enter__(self):
-        op = Operation(schema.Mutation)
-        op.start_local_job(job_name=self.name)
-        secret_uuid = send_graphql_request(op=op, json_response=False).start_local_job
-        variables.secret_uuid = secret_uuid
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        Batcher().wait_log_metrics()
-        if exc_type:
-            status = "FAILED"
-        else:
-            status = "SUCCESSFUL"
-        op = Operation(schema.Mutation)
-        op.stop_local_job(secret_uuid=variables.secret_uuid, status=status)
-        try:
-            _ = send_graphql_request(op=op, json_response=False).stop_local_job
-        finally:
-            variables.secret_uuid = None
-
-
-def start_local_job(
-    job_name, experiment_name: str = DEFAULT_EXPERIMENT, visibility=VisibilityOptions.PRIVATE
-) -> ActiveJob:
-    """
-    Create local job.
-
-    Parameters
-    ----------
-    job_name: str | None
-        Name of the new job. If not passed, it will be generated.
-    experiment_name: str = "Default"
-        Name of the experiment. Default: "Default"
-    visibility: VisibilityOptions
-        Visibility of this job to other users. Default: PRIVATE.
-
-    Returns
-    -------
-    ActiveJob
-        Active job.
-
-    Usage:
-        with start_local_job('my-beautiful-job') as job:
-            mlmanagement.log_metric(...)
-            mlmanagement.log_artifacts(...)
-    """
-    op = Operation(schema.Mutation)
-    create_local_job = op.create_local_job(
-        job_name=job_name, experiment_name=experiment_name, visibility=visibility.name
-    )
-    create_local_job.name()
-    create_local_job.visibility()
-    create_local_job.experiment.name()
-    job = send_graphql_request(op=op, json_response=True)
-    return ActiveJob(job["createLocalJob"])