trainml 0.5.8__py3-none-any.whl → 0.5.11__py3-none-any.whl

tests/unit/test_checkpoints_unit.py

@@ -44,9 +44,7 @@ class CheckpointsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await checkpoints.get("1234")
-        mock_trainml._query.assert_called_once_with(
-            "/checkpoint/1234", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/checkpoint/1234", "GET", dict())
 
     @mark.asyncio
     async def test_list_checkpoints(
@@ -57,9 +55,7 @@ class CheckpointsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await checkpoints.list()
-        mock_trainml._query.assert_called_once_with(
-            "/checkpoint", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/checkpoint", "GET", dict())
 
     @mark.asyncio
     async def test_remove_checkpoint(
@@ -86,11 +82,13 @@ class CheckpointsTests:
             name="new checkpoint",
             source_type="aws",
             source_uri="s3://trainml-examples/checkpoints/resnet50",
+            type="evefs",
         )
         api_response = {
             "project_uuid": "cus-id-1",
             "checkpoint_uuid": "checkpoint-id-1",
             "name": "new checkpoint",
+            "type": "evefs",
             "status": "new",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/checkpoints/resnet50",
@@ -135,7 +133,9 @@ class CheckpointTests:
 
     @mark.asyncio
     async def test_checkpoint_get_log_url(self, checkpoint, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await checkpoint.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -163,7 +163,9 @@ class CheckpointTests:
     async def test_checkpoint_get_connection_utility_url(
         self, checkpoint, mock_trainml
     ):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await checkpoint.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -338,9 +340,7 @@ class CheckpointTests:
         assert response.id == "data-id-1"
 
     @mark.asyncio
-    async def test_checkpoint_wait_for_successful(
-        self, checkpoint, mock_trainml
-    ):
+    async def test_checkpoint_wait_for_successful(self, checkpoint, mock_trainml):
         api_response = {
             "customer_uuid": "cus-id-1",
             "checkpoint_uuid": "data-id-1",
@@ -373,9 +373,7 @@ class CheckpointTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_checkpoint_wait_for_incorrect_status(
-        self, checkpoint, mock_trainml
-    ):
+    async def test_checkpoint_wait_for_incorrect_status(self, checkpoint, mock_trainml):
         api_response = None
         mock_trainml._query = AsyncMock(return_value=api_response)
         with raises(SpecificationError):
@@ -383,9 +381,7 @@ class CheckpointTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_checkpoint_wait_for_with_delay(
-        self, checkpoint, mock_trainml
-    ):
+    async def test_checkpoint_wait_for_with_delay(self, checkpoint, mock_trainml):
         api_response_initial = dict(
             checkpoint_uuid="1",
             name="first one",
@@ -441,9 +437,7 @@ class CheckpointTests:
         self, checkpoint, mock_trainml
     ):
         mock_trainml._query = AsyncMock(
-            side_effect=ApiError(
-                404, dict(errorMessage="Checkpoint Not Found")
-            )
+            side_effect=ApiError(404, dict(errorMessage="Checkpoint Not Found"))
         )
         await checkpoint.wait_for("archived")
         mock_trainml._query.assert_called()
@@ -453,9 +447,7 @@ class CheckpointTests:
         self, checkpoint, mock_trainml
     ):
         mock_trainml._query = AsyncMock(
-            side_effect=ApiError(
-                404, dict(errorMessage="Checkpoint Not Found")
-            )
+            side_effect=ApiError(404, dict(errorMessage="Checkpoint Not Found"))
         )
         with raises(ApiError):
             await checkpoint.wait_for("ready")

tests/unit/test_datasets_unit.py

@@ -28,6 +28,7 @@ def dataset(mock_trainml):
         dataset_uuid="1",
         project_uuid="proj-id-1",
         name="first one",
+        type="evefs",
         status="downloading",
         size=100000,
         createdAt="2020-12-31T23:59:59.000Z",
@@ -103,6 +104,7 @@ class DatasetsTests:
             name="new dataset",
             source_type="aws",
             source_uri="s3://trainml-examples/data/cifar10",
+            type="evefs",
         )
         api_response = {
             "customer_uuid": "cus-id-1",
@@ -110,6 +112,7 @@ class DatasetsTests:
             "dataset_uuid": "data-id-1",
             "name": "new dataset",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/data/cifar10",
             "createdAt": "2020-12-20T16:46:23.909Z",
@@ -139,9 +142,7 @@ class DatasetTests:
     def test_dataset_repr(self, dataset):
         string = repr(dataset)
         regex = (
-            r"^Dataset\( trainml , \*\*{.*'dataset_uuid': '"
-            + dataset.id
-            + r"'.*}\)$"
+            r"^Dataset\( trainml , \*\*{.*'dataset_uuid': '" + dataset.id + r"'.*}\)$"
         )
         assert isinstance(string, str)
         assert re.match(regex, string)
@@ -153,7 +154,9 @@ class DatasetTests:
 
     @mark.asyncio
     async def test_dataset_get_log_url(self, dataset, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await dataset.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -178,10 +181,10 @@ class DatasetTests:
         assert response == api_response
 
     @mark.asyncio
-    async def test_dataset_get_connection_utility_url(
-        self, dataset, mock_trainml
-    ):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+    async def test_dataset_get_connection_utility_url(self, dataset, mock_trainml):
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await dataset.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -388,9 +391,7 @@ class DatasetTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_incorrect_status(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_incorrect_status(self, dataset, mock_trainml):
         api_response = None
         mock_trainml._query = AsyncMock(return_value=api_response)
         with raises(SpecificationError):
@@ -435,9 +436,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_dataset_failed(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_dataset_failed(self, dataset, mock_trainml):
         api_response = dict(
             dataset_uuid="1",
             name="first one",
@@ -450,9 +449,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_archived_succeeded(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_archived_succeeded(self, dataset, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
         )
@@ -460,9 +457,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_unexpected_api_error(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_unexpected_api_error(self, dataset, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
         )

tests/unit/test_models_unit.py

@@ -28,6 +28,7 @@ def model(mock_trainml):
         model_uuid="1",
         project_uuid="proj-id-1",
         name="first one",
+        type="evefs",
         status="downloading",
         size=100000,
         createdAt="2020-12-31T23:59:59.000Z",
@@ -44,9 +45,7 @@ class ModelsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await models.get("1234")
-        mock_trainml._query.assert_called_once_with(
-            "/model/1234", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/model/1234", "GET", dict())
 
     @mark.asyncio
     async def test_list_models(
@@ -84,12 +83,14 @@ class ModelsTests:
             name="new model",
             source_type="aws",
             source_uri="s3://trainml-examples/models/resnet50",
+            type="evefs",
         )
         api_response = {
             "customer_uuid": "cus-id-1",
             "model_uuid": "model-id-1",
             "name": "new model",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/models/resnet50",
             "createdAt": "2020-12-20T16:46:23.909Z",
@@ -118,11 +119,7 @@ class ModelTests:
 
     def test_model_repr(self, model):
         string = repr(model)
-        regex = (
-            r"^Model\( trainml , \*\*{.*'model_uuid': '"
-            + model.id
-            + r"'.*}\)$"
-        )
+        regex = r"^Model\( trainml , \*\*{.*'model_uuid': '" + model.id + r"'.*}\)$"
         assert isinstance(string, str)
         assert re.match(regex, string)
 
@@ -133,7 +130,9 @@ class ModelTests:
 
     @mark.asyncio
     async def test_model_get_log_url(self, model, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await model.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -159,7 +158,9 @@ class ModelTests:
 
     @mark.asyncio
     async def test_model_get_connection_utility_url(self, model, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await model.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -425,9 +426,7 @@ class ModelTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_model_wait_for_archived_succeeded(
-        self, model, mock_trainml
-    ):
+    async def test_model_wait_for_archived_succeeded(self, model, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
         )
@@ -435,9 +434,7 @@ class ModelTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_model_wait_for_unexpected_api_error(
-        self, model, mock_trainml
-    ):
+    async def test_model_wait_for_unexpected_api_error(self, model, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
         )

tests/unit/test_volumes_unit.py

@@ -26,6 +26,7 @@ def volume(mock_trainml):
     yield specimen.Volume(
         mock_trainml,
         id="1",
+        type="evefs",
         project_uuid="proj-id-1",
         name="first one",
         status="downloading",
@@ -86,12 +87,14 @@ class VolumesTests:
             source_type="aws",
             source_uri="s3://trainml-examples/volumes/resnet50",
             capacity="10G",
+            type="evefs",
         )
         api_response = {
             "project_uuid": "cus-id-1",
             "id": "volume-id-1",
             "name": "new volume",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "capacity": "10G",
             "source_uri": "s3://trainml-examples/volumes/resnet50",

trainml/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.8"
+__version__ = "0.5.11"
 __all__ = "TrainML"

trainml/checkpoints.py

@@ -31,13 +31,24 @@ class Checkpoints(object):
         datasets = [Checkpoint(self.trainml, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.trainml.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.trainml.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Checkpoint {name}")

trainml/cli/cloudbender/datastore.py

@@ -29,13 +29,11 @@ def datastore(config):
 def list(config, provider, region):
     """List datastores."""
     data = [
-        ["ID", "NAME", "TYPE", "URI", "ROOT"],
+        ["ID", "NAME", "TYPE"],
         [
             "-" * 80,
             "-" * 80,
             "-" * 80,
-            "-" * 80,
-            "-" * 80,
         ],
     ]
 
@@ -51,15 +49,12 @@ def list(config, provider, region):
                 datastore.id,
                 datastore.name,
                 datastore.type,
-                datastore.uri,
-                datastore.root,
             ]
         )
 
     for row in data:
         click.echo(
-            "{: >37.36} {: >29.28} {: >9.8} {: >12.11} {: >12.11}"
-            "".format(*row),
+            "{: >37.36} {: >29.28} {: >9.8} " "".format(*row),
             file=config.stdout,
         )
 

trainml/cli/project/__init__.py

@@ -0,0 +1,84 @@
+import click
+from trainml.cli import cli, pass_config, search_by_id_name
+
+
+@cli.group()
+@pass_config
+def project(config):
+    """trainML project commands."""
+    pass
+
+
+@project.command()
+@pass_config
+def list(config):
+    """List projects."""
+    data = [
+        ["ID", "NAME", "OWNER", "MINE"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+
+    projects = config.trainml.run(config.trainml.client.projects.list())
+
+    for project in projects:
+        data.append(
+            [
+                project.id,
+                project.name,
+                project.owner_name,
+                "X" if project.is_owner else "",
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >4.4}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@project.command()
+@click.argument("name", type=click.STRING)
+@pass_config
+def create(config, name):
+    """
+    Create a project.
+
+    Project is created with the specified NAME.
+    """
+
+    return config.trainml.run(
+        config.trainml.client.projects.create(
+            name=name,
+        )
+    )
+
+
+@project.command()
+@click.argument("project", type=click.STRING)
+@pass_config
+def remove(config, project):
+    """
+    Remove a project.
+
+    PROJECT may be specified by name or ID, but ID is preferred.
+    """
+    projects = config.trainml.run(config.trainml.client.projects.list())
+
+    found = search_by_id_name(project, projects)
+    if None is found:
+        raise click.UsageError("Cannot find specified project.")
+
+    return config.trainml.run(found.remove())
+
+
+from trainml.cli.project.secret import secret
+from trainml.cli.project.key import key
+from trainml.cli.project.data_connector import data_connector
+from trainml.cli.project.datastore import datastore
+from trainml.cli.project.service import service

trainml/cli/project/data_connector.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from trainml.cli import pass_config
+from trainml.cli.project import project
+
+
+@project.group()
+@pass_config
+def data_connector(config):
+    """trainML project data_connector commands."""
+    pass
+
+
+@data_connector.command()
+@pass_config
+def list(config):
+    """List project data_connectors."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.trainml.run(
+        config.trainml.client.projects.get(config.trainml.client.project)
+    )
+
+    data_connectors = config.trainml.run(project.data_connectors.list())
+
+    for data_connector in data_connectors:
+        data.append(
+            [
+                data_connector.id,
+                data_connector.name,
+                data_connector.type,
+                data_connector.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@data_connector.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project data_connector list.
+    """
+    project = config.trainml.run(config.trainml.client.projects.get_current())
+
+    return config.trainml.run(project.data_connectors.refresh())

trainml/cli/project/datastore.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from trainml.cli import pass_config
+from trainml.cli.project import project
+
+
+@project.group()
+@pass_config
+def datastore(config):
+    """trainML project datastore commands."""
+    pass
+
+
+@datastore.command()
+@pass_config
+def list(config):
+    """List project datastores."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.trainml.run(
+        config.trainml.client.projects.get(config.trainml.client.project)
+    )
+
+    datastores = config.trainml.run(project.datastores.list())
+
+    for datastore in datastores:
+        data.append(
+            [
+                datastore.id,
+                datastore.name,
+                datastore.type,
+                datastore.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@datastore.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project datastore list.
+    """
+    project = config.trainml.run(config.trainml.client.projects.get_current())
+
+    return config.trainml.run(project.datastores.refresh())