trainml 0.5.8__py3-none-any.whl → 0.5.11__py3-none-any.whl

tests/integration/projects/conftest.py

@@ -0,0 +1,10 @@
+from pytest import fixture
+
+
+@fixture(scope="module")
+async def project(trainml):
+    project = await trainml.projects.create(
+        name="New Project", copy_keys=False, copy_secrets=False
+    )
+    yield project
+    await project.remove()

tests/integration/projects/test_projects_data_connectors_integration.py

@@ -0,0 +1,44 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.projects]
+
+
+@mark.create
+@mark.asyncio
+class ProjectDataConnectorsTests:
+    @fixture(scope="class")
+    async def project_data_connector(self, project):
+        data_connectors = await project.data_connectors.list()
+        yield data_connectors[0]
+
+    async def test_list_project_data_connectors(self, project, project_data_connector):
+        data_connectors = await project.data_connectors.list()
+        assert len(data_connectors) > 0
+
+    async def test_project_data_connector_properties(
+        self, project, project_data_connector
+    ):
+        assert isinstance(project_data_connector.project_uuid, str)
+        assert isinstance(project_data_connector.type, str)
+        assert isinstance(project_data_connector.name, str)
+        assert isinstance(project_data_connector.region_uuid, str)
+        assert project.id == project_data_connector.project_uuid
+
+    async def test_project_data_connector_str(self, project_data_connector):
+        string = str(project_data_connector)
+        regex = r"^{.*\"name\": \"" + project_data_connector.name + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_project_data_connector_repr(self, project_data_connector):
+        string = repr(project_data_connector)
+        regex = (
+            r"^ProjectDataConnector\( trainml , \*\*{.*'name': '"
+            + project_data_connector.name
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

tests/integration/projects/test_projects_datastores_integration.py

@@ -0,0 +1,42 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.projects]
+
+
+@mark.create
+@mark.asyncio
+class ProjectDatastoresTests:
+    @fixture(scope="class")
+    async def project_datastore(self, project):
+        datastores = await project.datastores.list()
+        yield datastores[0]
+
+    async def test_list_project_datastores(self, project):
+        datastores = await project.datastores.list()
+        assert len(datastores) > 0
+
+    async def test_project_datastore_properties(self, project, project_datastore):
+        assert isinstance(project_datastore.project_uuid, str)
+        assert isinstance(project_datastore.type, str)
+        assert isinstance(project_datastore.name, str)
+        assert isinstance(project_datastore.region_uuid, str)
+        assert project.id == project_datastore.project_uuid
+
+    async def test_project_datastore_str(self, project_datastore):
+        string = str(project_datastore)
+        regex = r"^{.*\"name\": \"" + project_datastore.name + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_project_datastore_repr(self, project_datastore):
+        string = repr(project_datastore)
+        regex = (
+            r"^ProjectDatastore\( trainml , \*\*{.*'name': '"
+            + project_datastore.name
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

tests/integration/{test_projects_integration.py → projects/test_projects_integration.py}

@@ -9,12 +9,6 @@ pytestmark = [mark.sdk, mark.integration, mark.projects]
 @mark.create
 @mark.asyncio
 class GetProjectsTests:
-    @fixture(scope="class")
-    async def project(self, trainml):
-        project = await trainml.projects.create(name="New Project", copy_keys=False)
-        yield project
-        await project.remove()
-
     async def test_get_projects(self, trainml):
         projects = await trainml.projects.list()
         assert len(projects) > 0

tests/integration/projects/test_projects_keys_integration.py

@@ -0,0 +1,43 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.projects]
+
+
+@mark.create
+@mark.asyncio
+class ProjectKeysTests:
+    @fixture(scope="class")
+    async def project_key(self, project):
+        project_key = await project.keys.put(
+            type="aws", key_id="ASFHALKF", secret="IUHKLHKAHF"
+        )
+        yield project_key
+        await project.keys.remove(type="aws")
+
+    async def test_list_project_keys(self, project, project_key):
+        keys = await project.keys.list()
+        assert len(keys) > 0
+
+    async def test_project_key_properties(self, project, project_key):
+        assert isinstance(project_key.project_uuid, str)
+        assert isinstance(project_key.type, str)
+        assert isinstance(project_key.key_id, str)
+        assert project_key.type == "aws"
+        assert project.id == project_key.project_uuid
+
+    async def test_project_key_str(self, project_key):
+        string = str(project_key)
+        regex = r"^{.*\"type\": \"" + project_key.type + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_project_key_repr(self, project_key):
+        string = repr(project_key)
+        regex = (
+            r"^ProjectKey\( trainml , \*\*{.*'type': '" + project_key.type + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

tests/integration/projects/test_projects_secrets_integration.py

@@ -0,0 +1,44 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.projects]
+
+
+@mark.create
+@mark.asyncio
+class ProjectSecretsTests:
+    @fixture(scope="class")
+    async def project_secret(self, project):
+        project_secret = await project.secrets.put(
+            name="secret_value", value="IUHKLHKAHF"
+        )
+        yield project_secret
+        await project.secrets.remove(name="secret_value")
+
+    async def test_list_project_keys(self, project, project_secret):
+        secrets = await project.secrets.list()
+        assert len(secrets) > 0
+
+    async def test_project_secret_properties(self, project, project_secret):
+        assert isinstance(project_secret.project_uuid, str)
+        assert isinstance(project_secret.name, str)
+        assert project_secret.name == "secret_value"
+        assert project.id == project_secret.project_uuid
+
+    async def test_project_secret_str(self, project_secret):
+        string = str(project_secret)
+        regex = r"^{.*\"name\": \"" + project_secret.name + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_project_secret_repr(self, project_secret):
+        string = repr(project_secret)
+        regex = (
+            r"^ProjectSecret\( trainml , \*\*{.*'name': '"
+            + project_secret.name
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

tests/integration/projects/test_projects_services_integration.py

@@ -0,0 +1,44 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.projects]
+
+
+@mark.create
+@mark.asyncio
+class ProjectServicesTests:
+    @fixture(scope="class")
+    async def project_service(self, project):
+        services = await project.services.list()
+        yield services[0]
+
+    async def test_list_project_services(self, project):
+        services = await project.services.list()
+        assert len(services) > 0
+
+    async def test_project_service_properties(self, project, project_service):
+        assert isinstance(project_service.project_uuid, str)
+        assert isinstance(project_service.name, str)
+        assert isinstance(project_service.id, str)
+        assert isinstance(project_service.hostname, str)
+        assert isinstance(project_service.region_uuid, str)
+        assert isinstance(project_service.public, bool)
+        assert project.id == project_service.project_uuid
+
+    async def test_project_service_str(self, project_service):
+        string = str(project_service)
+        regex = r"^{.*\"name\": \"" + project_service.name + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_project_service_repr(self, project_service):
+        string = repr(project_service)
+        regex = (
+            r"^ProjectService\( trainml , \*\*{.*'name': '"
+            + project_service.name
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

tests/integration/test_checkpoints_integration.py

@@ -16,7 +16,7 @@ class GetCheckpointTests:
             source_type="git",
             source_uri="git@github.com:trainML/environment-tests.git",
         )
-        checkpoint = await checkpoint.wait_for("ready", 120)
+        checkpoint = await checkpoint.wait_for("ready", 180)
         yield checkpoint
         await checkpoint.remove()
         checkpoint = await checkpoint.wait_for("archived", 60)
@@ -59,7 +59,6 @@ async def test_checkpoint_wasabi(trainml, capsys):
         name="CLI Automated Wasabi",
         source_type="wasabi",
         source_uri="s3://trainml-example/models/trainml-examples",
-        capacity="10G",
         source_options=dict(endpoint_url="https://s3.wasabisys.com"),
     )
     checkpoint = await checkpoint.wait_for("ready", 300)

tests/integration/test_models_integration.py

@@ -55,7 +55,6 @@ async def test_model_wasabi(trainml, capsys):
         name="CLI Automated Wasabi",
         source_type="wasabi",
         source_uri="s3://trainml-example/models/trainml-examples",
-        capacity="10G",
         source_options=dict(endpoint_url="https://s3.wasabisys.com"),
     )
     model = await model.wait_for("ready", 300)

tests/unit/cli/projects/test_cli_project_data_connector_unit.py

@@ -0,0 +1,28 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch, create_autospec
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.projects]
+
+from trainml.cli.project import data_connector as specimen
+from trainml.projects import (
+    Project,
+)
+
+
+def test_list_data_connectors(runner, mock_project_data_connectors):
+    with patch("trainml.cli.TrainML", new=AsyncMock) as mock_trainml:
+        project = create_autospec(Project)
+        mock_trainml.projects = AsyncMock()
+        mock_trainml.projects.get = AsyncMock(return_value=project)
+        mock_trainml.projects.get_current = AsyncMock(return_value=project)
+        project.data_connectors = AsyncMock()
+        project.data_connectors.list = AsyncMock(
+            return_value=mock_project_data_connectors
+        )
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        project.data_connectors.list.assert_called_once()

tests/unit/cli/projects/test_cli_project_datastore_unit.py

@@ -0,0 +1,26 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch, create_autospec
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.projects]
+
+from trainml.cli.project import datastore as specimen
+from trainml.projects import (
+    Project,
+)
+
+
+def test_list_datastores(runner, mock_project_datastores):
+    with patch("trainml.cli.TrainML", new=AsyncMock) as mock_trainml:
+        project = create_autospec(Project)
+        mock_trainml.projects = AsyncMock()
+        mock_trainml.projects.get = AsyncMock(return_value=project)
+        mock_trainml.projects.get_current = AsyncMock(return_value=project)
+        project.datastores = AsyncMock()
+        project.datastores.list = AsyncMock(return_value=mock_project_datastores)
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        project.datastores.list.assert_called_once()

tests/unit/cli/projects/test_cli_project_key_unit.py

@@ -0,0 +1,26 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch, create_autospec
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.projects]
+
+from trainml.cli.project import key as specimen
+from trainml.projects import (
+    Project,
+)
+
+
+def test_list_keys(runner, mock_project_keys):
+    with patch("trainml.cli.TrainML", new=AsyncMock) as mock_trainml:
+        project = create_autospec(Project)
+        mock_trainml.projects = AsyncMock()
+        mock_trainml.projects.get = AsyncMock(return_value=project)
+        mock_trainml.projects.get_current = AsyncMock(return_value=project)
+        project.keys = AsyncMock()
+        project.keys.list = AsyncMock(return_value=mock_project_keys)
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        project.keys.list.assert_called_once()

tests/unit/cli/projects/test_cli_project_secret_unit.py

@@ -0,0 +1,26 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch, create_autospec
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.projects]
+
+from trainml.cli.project import secret as specimen
+from trainml.projects import (
+    Project,
+)
+
+
+def test_list_secrets(runner, mock_project_secrets):
+    with patch("trainml.cli.TrainML", new=AsyncMock) as mock_trainml:
+        project = create_autospec(Project)
+        mock_trainml.projects = AsyncMock()
+        mock_trainml.projects.get = AsyncMock(return_value=project)
+        mock_trainml.projects.get_current = AsyncMock(return_value=project)
+        project.secrets = AsyncMock()
+        project.secrets.list = AsyncMock(return_value=mock_project_secrets)
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        project.secrets.list.assert_called_once()

tests/unit/cli/projects/test_cli_project_service_unit.py

@@ -0,0 +1,26 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch, create_autospec
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.projects]
+
+from trainml.cli.project import service as specimen
+from trainml.projects import (
+    Project,
+)
+
+
+def test_list_services(runner, mock_project_services):
+    with patch("trainml.cli.TrainML", new=AsyncMock) as mock_trainml:
+        project = create_autospec(Project)
+        mock_trainml.projects = AsyncMock()
+        mock_trainml.projects.get = AsyncMock(return_value=project)
+        mock_trainml.projects.get_current = AsyncMock(return_value=project)
+        project.services = AsyncMock()
+        project.services.list = AsyncMock(return_value=mock_project_services)
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        project.services.list.assert_called_once()

tests/unit/cli/projects/test_cli_project_unit.py

@@ -0,0 +1,19 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch, create_autospec
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.projects]
+
+from trainml.cli import project as specimen
+
+
+def test_list(runner, mock_projects):
+    with patch("trainml.cli.TrainML", new=AsyncMock) as mock_trainml:
+        mock_trainml.projects = AsyncMock()
+        mock_trainml.projects.list = AsyncMock(return_value=mock_projects)
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        mock_trainml.projects.list.assert_called_once()

tests/unit/cloudbender/test_datastores_unit.py

@@ -119,8 +119,6 @@ class datastoreTests:
         assert isinstance(datastore.region_uuid, str)
         assert isinstance(datastore.type, str)
         assert isinstance(datastore.name, str)
-        assert isinstance(datastore.uri, str)
-        assert isinstance(datastore.root, str)
 
     def test_datastore_str(self, datastore):
         string = str(datastore)
@@ -131,9 +129,7 @@ class datastoreTests:
     def test_datastore_repr(self, datastore):
         string = repr(datastore)
         regex = (
-            r"^Datastore\( trainml , \*\*{.*'store_id': '"
-            + datastore.id
-            + r"'.*}\)$"
+            r"^Datastore\( trainml , \*\*{.*'store_id': '" + datastore.id + r"'.*}\)$"
         )
         assert isinstance(string, str)
         assert re.match(regex, string)

tests/unit/conftest.py

@@ -16,9 +16,13 @@ from trainml.connections import Connections
 from trainml.projects import (
     Projects,
     Project,
-    ProjectDatastore,
-    ProjectService,
 )
+from trainml.projects.datastores import ProjectDatastores, ProjectDatastore
+from trainml.projects.services import ProjectServices, ProjectService
+from trainml.projects.data_connectors import ProjectDataConnectors, ProjectDataConnector
+from trainml.projects.keys import ProjectKeys, ProjectKey
+from trainml.projects.secrets import ProjectSecrets, ProjectSecret
+
 from trainml.cloudbender import Cloudbender
 from trainml.cloudbender.providers import Provider, Providers
 from trainml.cloudbender.regions import Region, Regions
@@ -26,6 +30,7 @@ from trainml.cloudbender.nodes import Node, Nodes
 from trainml.cloudbender.devices import Device, Devices
 from trainml.cloudbender.datastores import Datastore, Datastores
 from trainml.cloudbender.services import Service, Services
+from trainml.cloudbender.data_connectors import DataConnector, DataConnectors
 from trainml.cloudbender.device_configs import DeviceConfig, DeviceConfigs
 
 
@@ -948,6 +953,68 @@ def mock_project_services():
     ]
 
 
+@fixture(scope="session")
+def mock_data_connectors():
+    trainml = Mock()
+    yield [
+        DataConnector(
+            trainml,
+            **{
+                "provider_uuid": "prov-id-1",
+                "region_uuid": "reg-id-1",
+                "connector_id": "con-id-1",
+                "type": "custom",
+                "name": "On-Prem Connection A",
+                "protocol": "TCP",
+                "port_range": "8000-8099",
+                "cidr": "10.0.3.0/24",
+            },
+        ),
+        DataConnector(
+            trainml,
+            **{
+                "provider_uuid": "prov-id-2",
+                "region_uuid": "reg-id-2",
+                "connector_id": "con-id-2",
+                "type": "custom",
+                "name": "Cloud Connection B",
+                "protocol": "UDP",
+                "port_range": "5000",
+                "cidr": "10.0.2.0/24",
+            },
+        ),
+    ]
+
+
+@fixture(
+    scope="session",
+)
+def mock_project_data_connectors():
+    trainml = Mock()
+    yield [
+        ProjectDataConnector(
+            trainml,
+            **{
+                "project_uuid": "proj-id-1",
+                "region_uuid": "reg-id-1",
+                "id": "con-id-1",
+                "type": "custom",
+                "name": "On-Prem Connection A",
+            },
+        ),
+        ProjectDataConnector(
+            trainml,
+            **{
+                "project_uuid": "proj-id-1",
+                "region_uuid": "reg-id-2",
+                "id": "con-id-2",
+                "type": "custom",
+                "name": "Cloud Connection B",
+            },
+        ),
+    ]
+
+
 @fixture(scope="session")
 def mock_device_configs():
     trainml = Mock()
@@ -973,6 +1040,60 @@ def mock_device_configs():
     ]
 
 
+@fixture(
+    scope="session",
+)
+def mock_project_keys():
+    trainml = Mock()
+    yield [
+        ProjectKey(
+            trainml,
+            **{
+                "project_uuid": "proj-id-1",
+                "type": "aws",
+                "key_id": "AKSHFKHDFS",
+                "updatedAt": "2023-06-02T21:22:40.084Z",
+            },
+        ),
+        ProjectKey(
+            trainml,
+            **{
+                "project_uuid": "proj-id-1",
+                "type": "gcp",
+                "key_id": "credentials.json",
+                "updatedAt": "2023-06-02T21:22:40.084Z",
+            },
+        ),
+    ]
+
+
+@fixture(
+    scope="session",
+)
+def mock_project_secrets():
+    trainml = Mock()
+    yield [
+        ProjectSecret(
+            trainml,
+            **{
+                "project_uuid": "proj-id-1",
+                "name": "super_secret",
+                "created_by": "User",
+                "updatedAt": "2023-06-02T21:22:40.084Z",
+            },
+        ),
+        ProjectSecret(
+            trainml,
+            **{
+                "project_uuid": "proj-id-1",
+                "name": "super_secret_2",
+                "created_by": "User",
+                "updatedAt": "2023-06-02T21:22:40.084Z",
+            },
+        ),
+    ]
+
+
 @fixture(scope="function")
 def mock_trainml(
     mock_my_datasets,
@@ -991,7 +1112,13 @@ def mock_trainml(
     mock_devices,
     mock_datastores,
     mock_services,
+    mock_data_connectors,
     mock_device_configs,
+    mock_project_datastores,
+    mock_project_services,
+    mock_project_data_connectors,
+    mock_project_keys,
+    mock_project_secrets,
 ):
     trainml = create_autospec(TrainML)
     trainml.active_project = "proj-id-1"
@@ -1015,6 +1142,18 @@ def mock_trainml(
     trainml.environments.list = AsyncMock(return_value=mock_environments)
     trainml.jobs.list = AsyncMock(return_value=mock_jobs)
     trainml.projects.list = AsyncMock(return_value=mock_projects)
+    trainml.projects.datastores = create_autospec(ProjectDatastores)
+    trainml.projects.datastores.list = AsyncMock(return_value=mock_project_datastores)
+    trainml.projects.services = create_autospec(ProjectServices)
+    trainml.projects.services.list = AsyncMock(return_value=mock_project_services)
+    trainml.projects.data_connectors = create_autospec(ProjectDataConnectors)
+    trainml.projects.data_connectors.list = AsyncMock(
+        return_value=mock_project_data_connectors
+    )
+    trainml.projects.keys = create_autospec(ProjectKeys)
+    trainml.projects.keys.list = AsyncMock(return_value=mock_project_keys)
+    trainml.projects.secrets = create_autospec(ProjectSecrets)
+    trainml.projects.secrets.list = AsyncMock(return_value=mock_project_secrets)
 
     trainml.cloudbender = create_autospec(Cloudbender)
 
@@ -1024,12 +1163,16 @@ def mock_trainml(
     trainml.cloudbender.regions.list = AsyncMock(return_value=mock_regions)
     trainml.cloudbender.nodes = create_autospec(Nodes)
     trainml.cloudbender.nodes.list = AsyncMock(return_value=mock_nodes)
-    trainml.cloudbender.devices = create_autospec(Nodes)
+    trainml.cloudbender.devices = create_autospec(Devices)
     trainml.cloudbender.devices.list = AsyncMock(return_value=mock_devices)
     trainml.cloudbender.datastores = create_autospec(Datastores)
     trainml.cloudbender.datastores.list = AsyncMock(return_value=mock_datastores)
     trainml.cloudbender.services = create_autospec(Services)
     trainml.cloudbender.services.list = AsyncMock(return_value=mock_services)
+    trainml.cloudbender.data_connectors = create_autospec(DataConnectors)
+    trainml.cloudbender.data_connectors.list = AsyncMock(
+        return_value=mock_data_connectors
+    )
     trainml.cloudbender.device_configs = create_autospec(DeviceConfigs)
     trainml.cloudbender.device_configs.list = AsyncMock(
         return_value=mock_device_configs