trainml 0.5.9__py3-none-any.whl → 0.5.12__py3-none-any.whl

tests/unit/projects/test_projects_unit.py

@@ -75,7 +75,7 @@ class ProjectsTests:
         requested_config = dict(
             name="new project",
         )
-        expected_payload = dict(name="new project", copy_keys=False, copy_secrets=False)
+        expected_payload = dict(name="new project", copy_credentials=False)
         api_response = {
             "id": "project-id-1",
             "name": "new project",

tests/unit/test_checkpoints_unit.py

@@ -44,9 +44,7 @@ class CheckpointsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await checkpoints.get("1234")
-        mock_trainml._query.assert_called_once_with(
-            "/checkpoint/1234", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/checkpoint/1234", "GET", dict())
 
     @mark.asyncio
     async def test_list_checkpoints(
@@ -57,9 +55,7 @@ class CheckpointsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await checkpoints.list()
-        mock_trainml._query.assert_called_once_with(
-            "/checkpoint", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/checkpoint", "GET", dict())
 
     @mark.asyncio
     async def test_remove_checkpoint(
@@ -86,11 +82,13 @@ class CheckpointsTests:
             name="new checkpoint",
             source_type="aws",
             source_uri="s3://trainml-examples/checkpoints/resnet50",
+            type="evefs",
         )
         api_response = {
             "project_uuid": "cus-id-1",
             "checkpoint_uuid": "checkpoint-id-1",
             "name": "new checkpoint",
+            "type": "evefs",
             "status": "new",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/checkpoints/resnet50",
@@ -135,7 +133,9 @@ class CheckpointTests:
 
     @mark.asyncio
     async def test_checkpoint_get_log_url(self, checkpoint, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await checkpoint.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -163,7 +163,9 @@ class CheckpointTests:
     async def test_checkpoint_get_connection_utility_url(
         self, checkpoint, mock_trainml
     ):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await checkpoint.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -338,9 +340,7 @@ class CheckpointTests:
         assert response.id == "data-id-1"
 
     @mark.asyncio
-    async def test_checkpoint_wait_for_successful(
-        self, checkpoint, mock_trainml
-    ):
+    async def test_checkpoint_wait_for_successful(self, checkpoint, mock_trainml):
         api_response = {
             "customer_uuid": "cus-id-1",
             "checkpoint_uuid": "data-id-1",
@@ -373,9 +373,7 @@ class CheckpointTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_checkpoint_wait_for_incorrect_status(
-        self, checkpoint, mock_trainml
-    ):
+    async def test_checkpoint_wait_for_incorrect_status(self, checkpoint, mock_trainml):
         api_response = None
         mock_trainml._query = AsyncMock(return_value=api_response)
         with raises(SpecificationError):
@@ -383,9 +381,7 @@ class CheckpointTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_checkpoint_wait_for_with_delay(
-        self, checkpoint, mock_trainml
-    ):
+    async def test_checkpoint_wait_for_with_delay(self, checkpoint, mock_trainml):
         api_response_initial = dict(
             checkpoint_uuid="1",
             name="first one",
@@ -441,9 +437,7 @@ class CheckpointTests:
         self, checkpoint, mock_trainml
     ):
         mock_trainml._query = AsyncMock(
-            side_effect=ApiError(
-                404, dict(errorMessage="Checkpoint Not Found")
-            )
+            side_effect=ApiError(404, dict(errorMessage="Checkpoint Not Found"))
         )
         await checkpoint.wait_for("archived")
         mock_trainml._query.assert_called()
@@ -453,9 +447,7 @@ class CheckpointTests:
         self, checkpoint, mock_trainml
     ):
         mock_trainml._query = AsyncMock(
-            side_effect=ApiError(
-                404, dict(errorMessage="Checkpoint Not Found")
-            )
+            side_effect=ApiError(404, dict(errorMessage="Checkpoint Not Found"))
         )
         with raises(ApiError):
             await checkpoint.wait_for("ready")

tests/unit/test_datasets_unit.py

@@ -28,6 +28,7 @@ def dataset(mock_trainml):
         dataset_uuid="1",
         project_uuid="proj-id-1",
         name="first one",
+        type="evefs",
         status="downloading",
         size=100000,
         createdAt="2020-12-31T23:59:59.000Z",
@@ -103,6 +104,7 @@ class DatasetsTests:
             name="new dataset",
             source_type="aws",
             source_uri="s3://trainml-examples/data/cifar10",
+            type="evefs",
         )
         api_response = {
             "customer_uuid": "cus-id-1",
@@ -110,6 +112,7 @@ class DatasetsTests:
             "dataset_uuid": "data-id-1",
             "name": "new dataset",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/data/cifar10",
             "createdAt": "2020-12-20T16:46:23.909Z",
@@ -139,9 +142,7 @@ class DatasetTests:
     def test_dataset_repr(self, dataset):
         string = repr(dataset)
         regex = (
-            r"^Dataset\( trainml , \*\*{.*'dataset_uuid': '"
-            + dataset.id
-            + r"'.*}\)$"
+            r"^Dataset\( trainml , \*\*{.*'dataset_uuid': '" + dataset.id + r"'.*}\)$"
         )
         assert isinstance(string, str)
         assert re.match(regex, string)
@@ -153,7 +154,9 @@ class DatasetTests:
 
     @mark.asyncio
     async def test_dataset_get_log_url(self, dataset, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await dataset.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -178,10 +181,10 @@ class DatasetTests:
         assert response == api_response
 
     @mark.asyncio
-    async def test_dataset_get_connection_utility_url(
-        self, dataset, mock_trainml
-    ):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+    async def test_dataset_get_connection_utility_url(self, dataset, mock_trainml):
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await dataset.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -388,9 +391,7 @@ class DatasetTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_incorrect_status(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_incorrect_status(self, dataset, mock_trainml):
         api_response = None
         mock_trainml._query = AsyncMock(return_value=api_response)
         with raises(SpecificationError):
@@ -435,9 +436,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_dataset_failed(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_dataset_failed(self, dataset, mock_trainml):
         api_response = dict(
             dataset_uuid="1",
             name="first one",
@@ -450,9 +449,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_archived_succeeded(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_archived_succeeded(self, dataset, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
         )
@@ -460,9 +457,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_unexpected_api_error(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_unexpected_api_error(self, dataset, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
         )

tests/unit/test_models_unit.py

@@ -28,6 +28,7 @@ def model(mock_trainml):
         model_uuid="1",
         project_uuid="proj-id-1",
         name="first one",
+        type="evefs",
         status="downloading",
         size=100000,
         createdAt="2020-12-31T23:59:59.000Z",
@@ -44,9 +45,7 @@ class ModelsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await models.get("1234")
-        mock_trainml._query.assert_called_once_with(
-            "/model/1234", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/model/1234", "GET", dict())
 
     @mark.asyncio
     async def test_list_models(
@@ -84,12 +83,14 @@ class ModelsTests:
             name="new model",
             source_type="aws",
             source_uri="s3://trainml-examples/models/resnet50",
+            type="evefs",
         )
         api_response = {
             "customer_uuid": "cus-id-1",
             "model_uuid": "model-id-1",
             "name": "new model",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/models/resnet50",
             "createdAt": "2020-12-20T16:46:23.909Z",
@@ -118,11 +119,7 @@ class ModelTests:
 
     def test_model_repr(self, model):
         string = repr(model)
-        regex = (
-            r"^Model\( trainml , \*\*{.*'model_uuid': '"
-            + model.id
-            + r"'.*}\)$"
-        )
+        regex = r"^Model\( trainml , \*\*{.*'model_uuid': '" + model.id + r"'.*}\)$"
         assert isinstance(string, str)
         assert re.match(regex, string)
 
@@ -133,7 +130,9 @@ class ModelTests:
 
     @mark.asyncio
     async def test_model_get_log_url(self, model, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await model.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -159,7 +158,9 @@ class ModelTests:
 
     @mark.asyncio
     async def test_model_get_connection_utility_url(self, model, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await model.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -425,9 +426,7 @@ class ModelTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_model_wait_for_archived_succeeded(
-        self, model, mock_trainml
-    ):
+    async def test_model_wait_for_archived_succeeded(self, model, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
         )
@@ -435,9 +434,7 @@ class ModelTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_model_wait_for_unexpected_api_error(
-        self, model, mock_trainml
-    ):
+    async def test_model_wait_for_unexpected_api_error(self, model, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
         )

tests/unit/test_volumes_unit.py

@@ -26,6 +26,7 @@ def volume(mock_trainml):
     yield specimen.Volume(
         mock_trainml,
         id="1",
+        type="evefs",
         project_uuid="proj-id-1",
         name="first one",
         status="downloading",
@@ -86,12 +87,14 @@ class VolumesTests:
             source_type="aws",
             source_uri="s3://trainml-examples/volumes/resnet50",
             capacity="10G",
+            type="evefs",
         )
         api_response = {
             "project_uuid": "cus-id-1",
             "id": "volume-id-1",
             "name": "new volume",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "capacity": "10G",
             "source_uri": "s3://trainml-examples/volumes/resnet50",

trainml/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.9"
+__version__ = "0.5.12"
 __all__ = "TrainML"

trainml/checkpoints.py

@@ -31,13 +31,24 @@ class Checkpoints(object):
         datasets = [Checkpoint(self.trainml, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.trainml.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.trainml.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Checkpoint {name}")

trainml/cli/cloudbender/datastore.py

@@ -29,13 +29,11 @@ def datastore(config):
 def list(config, provider, region):
     """List datastores."""
     data = [
-        ["ID", "NAME", "TYPE", "URI", "ROOT"],
+        ["ID", "NAME", "TYPE"],
         [
             "-" * 80,
             "-" * 80,
             "-" * 80,
-            "-" * 80,
-            "-" * 80,
         ],
     ]
 
@@ -51,15 +49,12 @@ def list(config, provider, region):
                 datastore.id,
                 datastore.name,
                 datastore.type,
-                datastore.uri,
-                datastore.root,
             ]
         )
 
     for row in data:
         click.echo(
-            "{: >37.36} {: >29.28} {: >9.8} {: >12.11} {: >12.11}"
-            "".format(*row),
+            "{: >37.36} {: >29.28} {: >9.8} " "".format(*row),
             file=config.stdout,
         )
 

trainml/cli/job/create.py

@@ -172,12 +172,12 @@ def create(config):
     help="Conda packages to install as a comma separated list 'p1,\"p2=v2\",p3'",
 )
 @click.option(
-    "--key",
+    "--credential",
     type=click.Choice(
         ["aws", "gcp", "kaggle", "azure", "wasabi"],
         case_sensitive=False,
     ),
-    help="Third Party Keys to add to the job environment",
+    help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
 @click.option(
@@ -221,7 +221,7 @@ def notebook(
     environment,
     custom_image,
     env,
-    key,
+    credential,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -254,7 +254,7 @@ def notebook(
         data=dict(datasets=datasets),
         model=dict(checkpoints=checkpoints),
         environment=dict(
-            worker_key_types=[k for k in key],
+            credentials=[k for k in credential],
         ),
     )
 
@@ -500,12 +500,12 @@ def notebook(
     multiple=True,
 )
 @click.option(
-    "--key",
+    "--credential",
     type=click.Choice(
         ["aws", "gcp", "kaggle", "azure", "wasabi"],
         case_sensitive=False,
     ),
-    help="Third Party Keys to add to the job environment",
+    help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
 @click.option(
@@ -563,7 +563,7 @@ def training(
     environment,
     custom_image,
     env,
-    key,
+    credential,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -596,7 +596,7 @@ def training(
         data=dict(datasets=datasets),
         model=dict(checkpoints=checkpoints),
         environment=dict(
-            worker_key_types=[k for k in key],
+            credentials=[k for k in credential],
         ),
     )
 
@@ -844,12 +844,12 @@ def training(
     multiple=True,
 )
 @click.option(
-    "--key",
+    "--credential",
     type=click.Choice(
         ["aws", "gcp", "kaggle", "azure", "wasabi"],
         case_sensitive=False,
     ),
-    help="Third Party Keys to add to the job environment",
+    help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
 @click.option(
@@ -907,7 +907,7 @@ def inference(
     environment,
     custom_image,
     env,
-    key,
+    credential,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -933,7 +933,7 @@ def inference(
         data=dict(datasets=[]),
         model=dict(checkpoints=checkpoints),
         environment=dict(
-            worker_key_types=[k for k in key],
+            credentials=[k for k in credential],
         ),
     )
 
@@ -1163,12 +1163,12 @@ def from_json(config, attach, connect, file):
     multiple=True,
 )
 @click.option(
-    "--key",
+    "--credential",
     type=click.Choice(
         ["aws", "gcp", "kaggle", "azure", "wasabi"],
         case_sensitive=False,
     ),
-    help="Third Party Keys to add to the job environment",
+    help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
 @click.option(
@@ -1230,7 +1230,7 @@ def endpoint(
     environment,
     custom_image,
     env,
-    key,
+    credential,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -1257,7 +1257,7 @@ def endpoint(
         max_price=max_price,
         model=dict(checkpoints=checkpoints),
         environment=dict(
-            worker_key_types=[k for k in key],
+            credentials=[k for k in credential],
         ),
     )
 

trainml/cli/project/__init__.py

@@ -77,77 +77,8 @@ def remove(config, project):
     return config.trainml.run(found.remove())
 
 
-@project.command()
-@pass_config
-def list_datastores(config):
-    """List project datastores."""
-    data = [
-        ["ID", "NAME", "TYPE", "REGION_UUID"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.trainml.run(
-        config.trainml.client.projects.get(config.trainml.client.project)
-    )
-
-    datastores = config.trainml.run(project.list_datastores())
-
-    for datastore in datastores:
-        data.append(
-            [
-                datastore.id,
-                datastore.name,
-                datastore.type,
-                datastore.region_uuid,
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
-            file=config.stdout,
-        )
-
-
-@project.command()
-@pass_config
-def list_services(config):
-    """List project services."""
-    data = [
-        ["ID", "NAME", "HOSTNAME", "REGION_UUID"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.trainml.run(
-        config.trainml.client.projects.get(config.trainml.client.project)
-    )
-
-    services = config.trainml.run(project.list_services())
-
-    for service in services:
-        data.append(
-            [
-                service.id,
-                service.name,
-                service.hostname,
-                service.region_uuid,
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >38.36} {: >30.28} {: >30.28} {: >38.36}" "".format(*row),
-            file=config.stdout,
-        )
-
-
 from trainml.cli.project.secret import secret
-from trainml.cli.project.key import key
+from trainml.cli.project.credential import credential
+from trainml.cli.project.data_connector import data_connector
+from trainml.cli.project.datastore import datastore
+from trainml.cli.project.service import service