trainml 0.5.9__py3-none-any.whl → 0.5.11__py3-none-any.whl

tests/unit/test_datasets_unit.py

@@ -28,6 +28,7 @@ def dataset(mock_trainml):
         dataset_uuid="1",
         project_uuid="proj-id-1",
         name="first one",
+        type="evefs",
         status="downloading",
         size=100000,
         createdAt="2020-12-31T23:59:59.000Z",
@@ -103,6 +104,7 @@ class DatasetsTests:
             name="new dataset",
             source_type="aws",
             source_uri="s3://trainml-examples/data/cifar10",
+            type="evefs",
         )
         api_response = {
             "customer_uuid": "cus-id-1",
@@ -110,6 +112,7 @@ class DatasetsTests:
             "dataset_uuid": "data-id-1",
             "name": "new dataset",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/data/cifar10",
             "createdAt": "2020-12-20T16:46:23.909Z",
@@ -139,9 +142,7 @@ class DatasetTests:
     def test_dataset_repr(self, dataset):
         string = repr(dataset)
         regex = (
-            r"^Dataset\( trainml , \*\*{.*'dataset_uuid': '"
-            + dataset.id
-            + r"'.*}\)$"
+            r"^Dataset\( trainml , \*\*{.*'dataset_uuid': '" + dataset.id + r"'.*}\)$"
         )
         assert isinstance(string, str)
         assert re.match(regex, string)
@@ -153,7 +154,9 @@ class DatasetTests:
 
     @mark.asyncio
     async def test_dataset_get_log_url(self, dataset, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await dataset.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -178,10 +181,10 @@ class DatasetTests:
         assert response == api_response
 
     @mark.asyncio
-    async def test_dataset_get_connection_utility_url(
-        self, dataset, mock_trainml
-    ):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+    async def test_dataset_get_connection_utility_url(self, dataset, mock_trainml):
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await dataset.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -388,9 +391,7 @@ class DatasetTests:
         mock_trainml._query.assert_not_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_incorrect_status(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_incorrect_status(self, dataset, mock_trainml):
         api_response = None
         mock_trainml._query = AsyncMock(return_value=api_response)
         with raises(SpecificationError):
@@ -435,9 +436,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_dataset_failed(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_dataset_failed(self, dataset, mock_trainml):
         api_response = dict(
             dataset_uuid="1",
             name="first one",
@@ -450,9 +449,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_archived_succeeded(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_archived_succeeded(self, dataset, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
         )
@@ -460,9 +457,7 @@ class DatasetTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_dataset_wait_for_unexpected_api_error(
-        self, dataset, mock_trainml
-    ):
+    async def test_dataset_wait_for_unexpected_api_error(self, dataset, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
         )

tests/unit/test_models_unit.py

@@ -28,6 +28,7 @@ def model(mock_trainml):
         model_uuid="1",
         project_uuid="proj-id-1",
         name="first one",
+        type="evefs",
         status="downloading",
         size=100000,
         createdAt="2020-12-31T23:59:59.000Z",
@@ -44,9 +45,7 @@ class ModelsTests:
         api_response = dict()
         mock_trainml._query = AsyncMock(return_value=api_response)
         await models.get("1234")
-        mock_trainml._query.assert_called_once_with(
-            "/model/1234", "GET", dict()
-        )
+        mock_trainml._query.assert_called_once_with("/model/1234", "GET", dict())
 
     @mark.asyncio
     async def test_list_models(
@@ -84,12 +83,14 @@ class ModelsTests:
             name="new model",
             source_type="aws",
             source_uri="s3://trainml-examples/models/resnet50",
+            type="evefs",
         )
         api_response = {
             "customer_uuid": "cus-id-1",
             "model_uuid": "model-id-1",
             "name": "new model",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "source_uri": "s3://trainml-examples/models/resnet50",
             "createdAt": "2020-12-20T16:46:23.909Z",
@@ -118,11 +119,7 @@ class ModelTests:
 
     def test_model_repr(self, model):
         string = repr(model)
-        regex = (
-            r"^Model\( trainml , \*\*{.*'model_uuid': '"
-            + model.id
-            + r"'.*}\)$"
-        )
+        regex = r"^Model\( trainml , \*\*{.*'model_uuid': '" + model.id + r"'.*}\)$"
         assert isinstance(string, str)
         assert re.match(regex, string)
 
@@ -133,7 +130,9 @@ class ModelTests:
 
     @mark.asyncio
     async def test_model_get_log_url(self, model, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await model.get_log_url()
         mock_trainml._query.assert_called_once_with(
@@ -159,7 +158,9 @@ class ModelTests:
 
     @mark.asyncio
     async def test_model_get_connection_utility_url(self, model, mock_trainml):
-        api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        api_response = (
+            "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
+        )
         mock_trainml._query = AsyncMock(return_value=api_response)
         response = await model.get_connection_utility_url()
         mock_trainml._query.assert_called_once_with(
@@ -425,9 +426,7 @@ class ModelTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_model_wait_for_archived_succeeded(
-        self, model, mock_trainml
-    ):
+    async def test_model_wait_for_archived_succeeded(self, model, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
         )
@@ -435,9 +434,7 @@ class ModelTests:
         mock_trainml._query.assert_called()
 
     @mark.asyncio
-    async def test_model_wait_for_unexpected_api_error(
-        self, model, mock_trainml
-    ):
+    async def test_model_wait_for_unexpected_api_error(self, model, mock_trainml):
         mock_trainml._query = AsyncMock(
             side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
         )

tests/unit/test_volumes_unit.py

@@ -26,6 +26,7 @@ def volume(mock_trainml):
     yield specimen.Volume(
         mock_trainml,
         id="1",
+        type="evefs",
         project_uuid="proj-id-1",
         name="first one",
         status="downloading",
@@ -86,12 +87,14 @@ class VolumesTests:
             source_type="aws",
             source_uri="s3://trainml-examples/volumes/resnet50",
             capacity="10G",
+            type="evefs",
         )
         api_response = {
             "project_uuid": "cus-id-1",
             "id": "volume-id-1",
             "name": "new volume",
             "status": "new",
+            "type": "evefs",
             "source_type": "aws",
             "capacity": "10G",
             "source_uri": "s3://trainml-examples/volumes/resnet50",

trainml/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.9"
+__version__ = "0.5.11"
 __all__ = "TrainML"

trainml/checkpoints.py

@@ -31,13 +31,24 @@ class Checkpoints(object):
         datasets = [Checkpoint(self.trainml, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.trainml.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.trainml.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Checkpoint {name}")

trainml/cli/cloudbender/datastore.py

@@ -29,13 +29,11 @@ def datastore(config):
 def list(config, provider, region):
     """List datastores."""
     data = [
-        ["ID", "NAME", "TYPE", "URI", "ROOT"],
+        ["ID", "NAME", "TYPE"],
         [
             "-" * 80,
             "-" * 80,
             "-" * 80,
-            "-" * 80,
-            "-" * 80,
         ],
     ]
 
@@ -51,15 +49,12 @@ def list(config, provider, region):
                 datastore.id,
                 datastore.name,
                 datastore.type,
-                datastore.uri,
-                datastore.root,
             ]
         )
 
     for row in data:
         click.echo(
-            "{: >37.36} {: >29.28} {: >9.8} {: >12.11} {: >12.11}"
-            "".format(*row),
+            "{: >37.36} {: >29.28} {: >9.8} " "".format(*row),
             file=config.stdout,
         )
 

trainml/cli/project/__init__.py

@@ -77,77 +77,8 @@ def remove(config, project):
     return config.trainml.run(found.remove())
 
 
-@project.command()
-@pass_config
-def list_datastores(config):
-    """List project datastores."""
-    data = [
-        ["ID", "NAME", "TYPE", "REGION_UUID"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.trainml.run(
-        config.trainml.client.projects.get(config.trainml.client.project)
-    )
-
-    datastores = config.trainml.run(project.list_datastores())
-
-    for datastore in datastores:
-        data.append(
-            [
-                datastore.id,
-                datastore.name,
-                datastore.type,
-                datastore.region_uuid,
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
-            file=config.stdout,
-        )
-
-
-@project.command()
-@pass_config
-def list_services(config):
-    """List project services."""
-    data = [
-        ["ID", "NAME", "HOSTNAME", "REGION_UUID"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.trainml.run(
-        config.trainml.client.projects.get(config.trainml.client.project)
-    )
-
-    services = config.trainml.run(project.list_services())
-
-    for service in services:
-        data.append(
-            [
-                service.id,
-                service.name,
-                service.hostname,
-                service.region_uuid,
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >38.36} {: >30.28} {: >30.28} {: >38.36}" "".format(*row),
-            file=config.stdout,
-        )
-
-
 from trainml.cli.project.secret import secret
 from trainml.cli.project.key import key
+from trainml.cli.project.data_connector import data_connector
+from trainml.cli.project.datastore import datastore
+from trainml.cli.project.service import service

trainml/cli/project/data_connector.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from trainml.cli import pass_config
+from trainml.cli.project import project
+
+
+@project.group()
+@pass_config
+def data_connector(config):
+    """trainML project data_connector commands."""
+    pass
+
+
+@data_connector.command()
+@pass_config
+def list(config):
+    """List project data_connectors."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.trainml.run(
+        config.trainml.client.projects.get(config.trainml.client.project)
+    )
+
+    data_connectors = config.trainml.run(project.data_connectors.list())
+
+    for data_connector in data_connectors:
+        data.append(
+            [
+                data_connector.id,
+                data_connector.name,
+                data_connector.type,
+                data_connector.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@data_connector.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project data_connector list.
+    """
+    project = config.trainml.run(config.trainml.client.projects.get_current())
+
+    return config.trainml.run(project.data_connectors.refresh())

trainml/cli/project/datastore.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from trainml.cli import pass_config
+from trainml.cli.project import project
+
+
+@project.group()
+@pass_config
+def datastore(config):
+    """trainML project datastore commands."""
+    pass
+
+
+@datastore.command()
+@pass_config
+def list(config):
+    """List project datastores."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.trainml.run(
+        config.trainml.client.projects.get(config.trainml.client.project)
+    )
+
+    datastores = config.trainml.run(project.datastores.list())
+
+    for datastore in datastores:
+        data.append(
+            [
+                datastore.id,
+                datastore.name,
+                datastore.type,
+                datastore.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@datastore.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project datastore list.
+    """
+    project = config.trainml.run(config.trainml.client.projects.get_current())
+
+    return config.trainml.run(project.datastores.refresh())

trainml/cli/project/service.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from trainml.cli import pass_config
+from trainml.cli.project import project
+
+
+@project.group()
+@pass_config
+def service(config):
+    """trainML project service commands."""
+    pass
+
+
+@service.command()
+@pass_config
+def list(config):
+    """List project services."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.trainml.run(
+        config.trainml.client.projects.get(config.trainml.client.project)
+    )
+
+    services = config.trainml.run(project.services.list())
+
+    for service in services:
+        data.append(
+            [
+                service.id,
+                service.name,
+                service.hostname,
+                service.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@service.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project service list.
+    """
+    project = config.trainml.run(config.trainml.client.projects.get_current())
+
+    return config.trainml.run(project.services.refresh())

trainml/cloudbender/data_connectors.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from trainml.exceptions import (
+    ApiError,
+    SpecificationError,
+    TrainMLException,
+)
 
 
 class DataConnectors(object):

trainml/cloudbender/datastores.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from trainml.exceptions import (
+    ApiError,
+    SpecificationError,
+    TrainMLException,
+)
 
 
 class Datastores(object):
@@ -20,9 +28,7 @@ class Datastores(object):
             "GET",
             kwargs,
         )
-        datastores = [
-            Datastore(self.trainml, **datastore) for datastore in resp
-        ]
+        datastores = [Datastore(self.trainml, **datastore) for datastore in resp]
         return datastores
 
     async def create(
@@ -31,18 +37,12 @@ class Datastores(object):
         region_uuid,
         name,
         type,
-        uri,
-        root,
-        options=None,
         **kwargs,
     ):
         logging.info(f"Creating Datastore {name}")
         data = dict(
             name=name,
             type=type,
-            uri=uri,
-            root=root,
-            options=options,
             **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
@@ -73,8 +73,6 @@ class Datastore:
         self._region_uuid = self._datastore.get("region_uuid")
         self._type = self._datastore.get("type")
         self._name = self._datastore.get("name")
-        self._uri = self._datastore.get("uri")
-        self._root = self._datastore.get("root")
 
     @property
     def id(self) -> str:
@@ -96,14 +94,6 @@ class Datastore:
     def name(self) -> str:
         return self._name
 
-    @property
-    def uri(self) -> str:
-        return self._uri
-
-    @property
-    def root(self) -> str:
-        return self._root
-
     def __str__(self):
         return json.dumps({k: v for k, v in self._datastore.items()})
 

trainml/cloudbender/nodes.py

@@ -1,5 +1,9 @@
 import json
 import logging
+import asyncio
+import math
+
+from trainml.exceptions import ApiError, SpecificationError, TrainMLException, NodeError
 
 
 class Nodes(object):
@@ -29,7 +33,7 @@ class Nodes(object):
         region_uuid,
         friendly_name,
         hostname,
-        minion_id,
+        minion_id=None,
         type="permanent",
         service="compute",
         **kwargs,
@@ -153,3 +157,42 @@ class Node:
             None,
             dict(command=command),
         )
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["active", "maintenance", "offline", "stopped", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status in ["errored", "failed"]:
+                raise NodeError(self.status, self)
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise TrainMLException(f"Timeout waiting for {status}")