proximl 0.5.13__py3-none-any.whl → 0.5.14__py3-none-any.whl

tests/integration/projects/conftest.py

@@ -4,7 +4,7 @@ from pytest import fixture
 @fixture(scope="module")
 async def project(proximl):
     project = await proximl.projects.create(
-        name="New Project", copy_keys=False, copy_secrets=False
+        name="New Project", copy_credentials=False, copy_secrets=False
     )
     yield project
     await project.remove()

tests/integration/projects/test_projects_members_integration.py

@@ -0,0 +1,49 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.projects]
+
+
+@mark.create
+@mark.asyncio
+class ProjectMembersTests:
+    @fixture(scope="class")
+    async def project_member(self, project):
+        member = await project.members.add("test.account@proximl.ai","read","read","read","read","read")
+        yield member
+        await project.members.remove("test.account@proximl.ai")
+
+    async def test_list_project_members(self, project):
+        members = await project.members.list()
+        assert len(members) > 0
+
+    async def test_project_member_properties(self, project, project_member):
+        assert isinstance(project_member.id, str)
+        assert isinstance(project_member.email, str)
+        assert isinstance(project_member.project_uuid, str)
+        assert isinstance(project_member.owner, bool)
+        assert isinstance(project_member.job, str)
+        assert isinstance(project_member.dataset, str)
+        assert isinstance(project_member.model, str)
+        assert isinstance(project_member.checkpoint, str)
+        assert isinstance(project_member.volume, str)
+        assert project.id == project_member.project_uuid
+        assert project_member.id == "test.account@proximl.ai"
+
+    async def test_project_member_str(self, project_member):
+        string = str(project_member)
+        regex = r"^{.*\"email\": \"" + project_member.email + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_project_member_repr(self, project_member):
+        string = repr(project_member)
+        regex = (
+            r"^ProjectMember\( proximl , \*\*{.*'email': '"
+            + project_member.email
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

tests/integration/projects/test_projects_secrets_integration.py

@@ -17,7 +17,7 @@ class ProjectSecretsTests:
         yield project_secret
         await project.secrets.remove(name="secret_value")
 
-    async def test_list_project_keys(self, project, project_secret):
+    async def test_list_project_secrets(self, project, project_secret):
         secrets = await project.secrets.list()
         assert len(secrets) > 0
 

tests/integration/test_jobs_integration.py

@@ -175,7 +175,7 @@ class JobAPIResourceValidationTests:
                 disk_size=10,
             )
         assert (
-            "Invalid Request - None (CPU Only) may be not be combined with other GPU Types"
+            "Invalid Request - CPU Only may be not be combined with other GPU Types"
             in error.value.message
         )
 
@@ -423,11 +423,11 @@ class JobIOTests:
             disk_size=10,
             workers=["python $ML_MODEL_PATH/tensorflow/main.py"],
             environment=dict(
-                type="DEEPLEARNING_PY310",
+                type="DEEPLEARNING_PY312",
                 env=[
                     dict(
                         key="CHECKPOINT_FILE",
-                        value="model.ckpt-0050",
+                        value="model.ckpt-0050.weights.h5",
                     )
                 ],
             ),
@@ -469,7 +469,7 @@ class JobIOTests:
         sys.stderr.write(captured.err)
         assert "Epoch 1/2" in captured.out
         assert "Epoch 2/2" in captured.out
-        assert "adding: model.ckpt-0001.data-00000-of-00001" in captured.out
+        assert "adding: model.ckpt-0001" in captured.out
         assert "Send complete" in captured.out
 
     async def test_job_model_input_and_output(self, proximl, capsys):
@@ -558,7 +558,7 @@ class JobTypeTests:
         assert job.url
         assert extract_domain_suffix(urlparse(job.url).hostname) == "proximl.cloud"
         tries = 0
-        await asyncio.sleep(30)
+        await asyncio.sleep(180) ## downloading weights can be slow
         async with aiohttp.ClientSession() as session:
             retry = True
             while retry:
@@ -640,7 +640,7 @@ class JobTypeTests:
         assert "Epoch 2/2" in captured.out
         assert "Uploading s3://proximl-example/output/resnet_cifar10" in captured.out
         assert (
-            "upload: ./model.ckpt-0002.data-00000-of-00001 to s3://proximl-example/output/resnet_cifar10/model.ckpt-0002.data-00000-of-00001"
+            "upload: ./model.ckpt-0002.weights.h5 to s3://proximl-example/output/resnet_cifar10/model.ckpt-0002.weights.h5"
             in captured.out
         )
         assert "Upload complete" in captured.out
@@ -713,9 +713,9 @@ class JobFeatureTests:
         sys.stderr.write(captured.err)
         upload_contents = os.listdir(temp_dir.name)
         temp_dir.cleanup()
-        assert len(upload_contents) > 4
+        assert len(upload_contents) >= 3
         assert any(
-            "model.ckpt-0002.data-00000-of-00001" in content
+            "model.ckpt-0002" in content
             for content in upload_contents
         )
 
@@ -724,5 +724,5 @@ class JobFeatureTests:
         sys.stderr.write(captured.err)
         assert "Epoch 1/2" in captured.out
         assert "Epoch 2/2" in captured.out
-        assert "Number of regular files transferred: 7" in captured.out
+        assert "Number of regular files transferred: 4" in captured.out
         assert "Send complete" in captured.out

tests/unit/conftest.py

@@ -20,7 +20,7 @@ from proximl.projects import (
 from proximl.projects.datastores import ProjectDatastores, ProjectDatastore
 from proximl.projects.services import ProjectServices, ProjectService
 from proximl.projects.data_connectors import ProjectDataConnectors, ProjectDataConnector
-from proximl.projects.keys import ProjectKeys, ProjectKey
+from proximl.projects.credentials import ProjectCredentials, ProjectCredential
 from proximl.projects.secrets import ProjectSecrets, ProjectSecret
 
 from proximl.cloudbender import Cloudbender
@@ -532,7 +532,7 @@ def mock_jobs():
                         {"value": "env1val", "key": "env1"},
                         {"value": "env2val", "key": "env2"},
                     ],
-                    "worker_key_types": ["aws", "gcp"],
+                    "credentials": ["aws", "gcp"],
                     "status": "new",
                 },
                 "vpn": {
@@ -609,7 +609,7 @@ def mock_jobs():
                     "type": "DEEPLEARNING_PY37",
                     "image_size": 39656398629,
                     "env": [],
-                    "worker_key_types": [],
+                    "credentials": [],
                     "status": "ready",
                 },
                 "vpn": {
@@ -1043,10 +1043,10 @@ def mock_device_configs():
 @fixture(
     scope="session",
 )
-def mock_project_keys():
+def mock_project_credentials():
     proximl = Mock()
     yield [
-        ProjectKey(
+        ProjectCredential(
             proximl,
             **{
                 "project_uuid": "proj-id-1",
@@ -1055,7 +1055,7 @@ def mock_project_keys():
                 "updatedAt": "2023-06-02T21:22:40.084Z",
             },
         ),
-        ProjectKey(
+        ProjectCredential(
             proximl,
             **{
                 "project_uuid": "proj-id-1",
@@ -1117,7 +1117,7 @@ def mock_proximl(
     mock_project_datastores,
     mock_project_services,
     mock_project_data_connectors,
-    mock_project_keys,
+    mock_project_credentials,
     mock_project_secrets,
 ):
     proximl = create_autospec(ProxiML)
@@ -1150,8 +1150,8 @@ def mock_proximl(
     proximl.projects.data_connectors.list = AsyncMock(
         return_value=mock_project_data_connectors
     )
-    proximl.projects.keys = create_autospec(ProjectKeys)
-    proximl.projects.keys.list = AsyncMock(return_value=mock_project_keys)
+    proximl.projects.credentials = create_autospec(ProjectCredentials)
+    proximl.projects.credentials.list = AsyncMock(return_value=mock_project_credentials)
     proximl.projects.secrets = create_autospec(ProjectSecrets)
     proximl.projects.secrets.list = AsyncMock(return_value=mock_project_secrets)
 

tests/unit/projects/test_project_members_unit.py

@@ -0,0 +1,107 @@
+import re
+import json
+import logging
+from unittest.mock import AsyncMock, patch
+from pytest import mark, fixture, raises
+from aiohttp import WSMessage, WSMsgType
+
+import proximl.projects.members as specimen
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
+
+pytestmark = [mark.sdk, mark.unit, mark.projects]
+
+
+@fixture
+def project_members(mock_proximl):
+    yield specimen.ProjectMembers(mock_proximl, project_id="1")
+
+
+@fixture
+def project_member(mock_proximl):
+    yield specimen.ProjectMember(
+        mock_proximl,
+        id="owner@gmail.com",
+        email="owner@gmail.com",
+        project_uuid="proj-id-1",
+       owner= True,
+                job= "all",
+                dataset= "all",
+                model= "all",
+                checkpoint="all",
+                volume= "all"
+    )
+
+
+class ProjectMembersTests:
+    @mark.asyncio
+    async def test_project_members_list(self, project_members, mock_proximl):
+        api_response = [
+            {
+                "project_uuid": "proj-id-1",
+                "email": "owner@gmail.com",
+                "createdAt": "2024-09-04T00:42:39.529Z",
+                "updatedAt": "2024-09-04T00:42:39.529Z",
+                "owner": True,
+                "job": "all",
+                "dataset": "all",
+                "model": "all",
+                "checkpoint": "all",
+                "volume": "all"
+            },
+            {
+                "project_uuid": "proj-id-1",
+                "email": "non-owner@gmail.com",
+                "createdAt": "2024-09-04T00:42:39.529Z",
+                "updatedAt": "2024-09-04T00:42:39.529Z",
+                "owner": False,
+                "job": "all",
+                "dataset": "all",
+                "model": "all",
+                "checkpoint": "read",
+                "volume": "read"
+            },
+        ]
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        resp = await project_members.list()
+        mock_proximl._query.assert_called_once_with(
+            "/project/1/access", "GET", dict()
+        )
+        assert len(resp) == 2
+
+
+class ProjectMemberTests:
+    def test_project_member_properties(self, project_member):
+        assert isinstance(project_member.id, str)
+        assert isinstance(project_member.email, str)
+        assert isinstance(project_member.project_uuid, str)
+        assert isinstance(project_member.owner, bool)
+        assert isinstance(project_member.job, str)
+        assert isinstance(project_member.dataset, str)
+        assert isinstance(project_member.model, str)
+        assert isinstance(project_member.checkpoint, str)
+        assert isinstance(project_member.volume, str)
+
+    def test_project_member_str(self, project_member):
+        string = str(project_member)
+        regex = r"^{.*\"id\": \"" + project_member.id + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    def test_project_member_repr(self, project_member):
+        string = repr(project_member)
+        regex = (
+            r"^ProjectMember\( proximl , \*\*{.*'id': '"
+            + project_member.id
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    def test_project_member_bool(self, project_member, mock_proximl):
+        empty_project_member = specimen.ProjectMember(mock_proximl)
+        assert bool(project_member)
+        assert not bool(empty_project_member)

tests/unit/projects/test_projects_unit.py

@@ -75,7 +75,7 @@ class ProjectsTests:
         requested_config = dict(
             name="new project",
         )
-        expected_payload = dict(name="new project", copy_keys=False, copy_secrets=False)
+        expected_payload = dict(name="new project", copy_credentials=False)
         api_response = {
             "id": "project-id-1",
             "name": "new project",

proximl/cli/project/key.py

@@ -1,124 +0,0 @@
-import click
-import os
-import json
-import base64
-from pathlib import Path
-from proximl.cli import pass_config
-from proximl.cli.project import project
-
-
-@project.group()
-@pass_config
-def key(config):
-    """proxiML project key commands."""
-    pass
-
-
-@key.command()
-@pass_config
-def list(config):
-    """List keys."""
-    data = [
-        ["TYPE", "KEY ID", "UPDATED AT"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.proximl.run(config.proximl.client.projects.get_current())
-    keys = config.proximl.run(project.keys.list())
-
-    for key in keys:
-        data.append(
-            [
-                key.type,
-                key.key_id,
-                key.updated_at.isoformat(timespec="seconds"),
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >13.11} {: >37.35} {: >28.26}" "".format(*row),
-            file=config.stdout,
-        )
-
-
-@key.command()
-@click.argument(
-    "type",
-    type=click.Choice(
-        [
-            "aws",
-            "azure",
-            "docker",
-            "gcp",
-            "huggingface",
-            "kaggle",
-            "ngc",
-            "wasabi",
-        ],
-        case_sensitive=False,
-    ),
-)
-@pass_config
-def put(config, type):
-    """
-    Set a key.
-
-    A key is uploaded.
-    """
-    project = config.proximl.run(config.proximl.client.projects.get_current())
-
-    tenant = None
-
-    if type in ["aws", "wasabi"]:
-        key_id = click.prompt("Enter the key ID", type=str, hide_input=False)
-        secret = click.prompt("Enter the secret key", type=str, hide_input=True)
-    elif type == "azure":
-        key_id = click.prompt(
-            "Enter the Application (client) ID", type=str, hide_input=False
-        )
-        tenant = click.prompt(
-            "Enter the Directory (tenant) ley", type=str, hide_input=False
-        )
-        secret = click.prompt("Enter the client secret", type=str, hide_input=True)
-    elif type in ["docker", "huggingface"]:
-        key_id = click.prompt("Enter the username", type=str, hide_input=False)
-        secret = click.prompt("Enter the access token", type=str, hide_input=True)
-    elif type in ["gcp", "kaggle"]:
-        file_name = click.prompt(
-            "Enter the path of the credentials file",
-            type=click.Path(
-                exists=True, file_okay=True, dir_okay=False, resolve_path=True
-            ),
-            hide_input=False,
-        )
-        key_id = os.path.basename(file_name)
-        with open(file_name) as f:
-            secret = json.load(f)
-        secret = json.dumps(secret)
-    elif type == "ngc":
-        key_id = "$oauthtoken"
-        secret = click.prompt("Enter the access token", type=str, hide_input=True)
-    else:
-        raise click.UsageError("Unsupported key type")
-
-    return config.proximl.run(
-        project.keys.put(type=type, key_id=key_id, secret=secret, tenant=tenant)
-    )
-
-
-@key.command()
-@click.argument("name", type=click.STRING)
-@pass_config
-def remove(config, name):
-    """
-    Remove a key.
-
-
-    """
-    project = config.proximl.run(config.proximl.client.projects.get_current())
-
-    return config.proximl.run(project.key.remove(name))

proximl/projects/keys.py

@@ -1,71 +0,0 @@
-import json
-import logging
-from datetime import datetime
-from dateutil import parser, tz
-
-
-class ProjectKeys(object):
-    def __init__(self, proximl, project_id):
-        self.proximl = proximl
-        self.project_id = project_id
-
-    async def list(self, **kwargs):
-        resp = await self.proximl._query(
-            f"/project/{self.project_id}/keys", "GET", kwargs
-        )
-        keys = [ProjectKey(self.proximl, **service) for service in resp]
-        return keys
-
-    async def put(self, type, key_id, secret, tenant=None, **kwargs):
-        data = dict(key_id=key_id, secret=secret, tenant=tenant)
-        payload = {k: v for k, v in data.items() if v is not None}
-        logging.info(f"Creating Project Key {type}")
-        resp = await self.proximl._query(
-            f"/project/{self.project_id}/key/{type}", "PUT", None, payload
-        )
-        key = ProjectKey(self.proximl, **resp)
-        logging.info(f"Created Project Key {type} in project {self.project_id}")
-
-        return key
-
-    async def remove(self, type, **kwargs):
-        await self.proximl._query(
-            f"/project/{self.project_id}/key/{type}", "DELETE", kwargs
-        )
-
-
-class ProjectKey:
-    def __init__(self, proximl, **kwargs):
-        self.proximl = proximl
-        self._entity = kwargs
-        self._type = self._entity.get("type")
-        self._project_uuid = self._entity.get("project_uuid")
-        self._key_id = self._entity.get("key_id")
-        self._updated_at = self._entity.get("updatedAt")
-
-    @property
-    def type(self) -> str:
-        return self._type
-
-    @property
-    def project_uuid(self) -> str:
-        return self._project_uuid
-
-    @property
-    def key_id(self) -> str:
-        return self._key_id
-
-    @property
-    def updated_at(self) -> datetime:
-        timestamp = parser.isoparse(self._updated_at)
-        timezone = tz.tzlocal()
-        return timestamp.astimezone(timezone)
-
-    def __str__(self):
-        return json.dumps({k: v for k, v in self._entity.items()})
-
-    def __repr__(self):
-        return f"ProjectKey( proximl , **{self._entity.__repr__()})"
-
-    def __bool__(self):
-        return bool(self._type)

tests/integration/projects/test_projects_keys_integration.py

@@ -1,43 +0,0 @@
-import re
-import sys
-import asyncio
-from pytest import mark, fixture
-
-pytestmark = [mark.sdk, mark.integration, mark.projects]
-
-
-@mark.create
-@mark.asyncio
-class ProjectKeysTests:
-    @fixture(scope="class")
-    async def project_key(self, project):
-        project_key = await project.keys.put(
-            type="aws", key_id="ASFHALKF", secret="IUHKLHKAHF"
-        )
-        yield project_key
-        await project.keys.remove(type="aws")
-
-    async def test_list_project_keys(self, project, project_key):
-        keys = await project.keys.list()
-        assert len(keys) > 0
-
-    async def test_project_key_properties(self, project, project_key):
-        assert isinstance(project_key.project_uuid, str)
-        assert isinstance(project_key.type, str)
-        assert isinstance(project_key.key_id, str)
-        assert project_key.type == "aws"
-        assert project.id == project_key.project_uuid
-
-    async def test_project_key_str(self, project_key):
-        string = str(project_key)
-        regex = r"^{.*\"type\": \"" + project_key.type + r"\".*}$"
-        assert isinstance(string, str)
-        assert re.match(regex, string)
-
-    async def test_project_key_repr(self, project_key):
-        string = repr(project_key)
-        regex = (
-            r"^ProjectKey\( proximl , \*\*{.*'type': '" + project_key.type + r"'.*}\)$"
-        )
-        assert isinstance(string, str)
-        assert re.match(regex, string)

tests/unit/cli/projects/test_cli_project_key_unit.py

@@ -1,26 +0,0 @@
-import re
-import json
-import click
-from unittest.mock import AsyncMock, patch, create_autospec
-from pytest import mark, fixture, raises
-
-pytestmark = [mark.cli, mark.unit, mark.projects]
-
-from proximl.cli.project import key as specimen
-from proximl.projects import (
-    Project,
-)
-
-
-def test_list_keys(runner, mock_project_keys):
-    with patch("proximl.cli.ProxiML", new=AsyncMock) as mock_proximl:
-        project = create_autospec(Project)
-        mock_proximl.projects = AsyncMock()
-        mock_proximl.projects.get = AsyncMock(return_value=project)
-        mock_proximl.projects.get_current = AsyncMock(return_value=project)
-        project.keys = AsyncMock()
-        project.keys.list = AsyncMock(return_value=mock_project_keys)
-        result = runner.invoke(specimen, ["list"])
-        print(result)
-        assert result.exit_code == 0
-        project.keys.list.assert_called_once()

tests/unit/projects/test_project_keys_unit.py

@@ -1,96 +0,0 @@
-import re
-import json
-import logging
-from unittest.mock import AsyncMock, patch
-from pytest import mark, fixture, raises
-from aiohttp import WSMessage, WSMsgType
-
-import proximl.projects.keys as specimen
-from proximl.exceptions import (
-    ApiError,
-    SpecificationError,
-    ProxiMLException,
-)
-
-pytestmark = [mark.sdk, mark.unit, mark.projects]
-
-
-@fixture
-def project_keys(mock_proximl):
-    yield specimen.ProjectKeys(mock_proximl, project_id="1")
-
-
-@fixture
-def project_key(mock_proximl):
-    yield specimen.ProjectKey(
-        mock_proximl, project_uuid="proj-id-1", type="aws", key_id="AIYHGFSDLK"
-    )
-
-
-class ProjectKeysTests:
-    @mark.asyncio
-    async def test_project_keys_list(self, project_keys, mock_proximl):
-        api_response = [
-            {"project_uuid": "proj-id-1", "type": "aws", "key_id": "AIYHGFSDLK"},
-            {"project_uuid": "proj-id-1", "type": "gcp", "key_id": "credentials.json"},
-        ]
-        mock_proximl._query = AsyncMock(return_value=api_response)
-        resp = await project_keys.list()
-        mock_proximl._query.assert_called_once_with("/project/1/keys", "GET", dict())
-        assert len(resp) == 2
-
-    @mark.asyncio
-    async def test_remove_project_key(
-        self,
-        project_keys,
-        mock_proximl,
-    ):
-        api_response = dict()
-        mock_proximl._query = AsyncMock(return_value=api_response)
-        await project_keys.remove("aws")
-        mock_proximl._query.assert_called_once_with(
-            "/project/1/key/aws", "DELETE", dict()
-        )
-
-    @mark.asyncio
-    async def test_put_project_key(self, project_keys, mock_proximl):
-        requested_config = dict(type="aws", key_id="AIUDHADA", secret="ASKHJSLKF")
-        expected_payload = dict(key_id="AIUDHADA", secret="ASKHJSLKF")
-        api_response = {
-            "project_uuid": "project-id-1",
-            "type": "aws",
-            "key_id": "AIUDHADA",
-        }
-
-        mock_proximl._query = AsyncMock(return_value=api_response)
-        response = await project_keys.put(**requested_config)
-        mock_proximl._query.assert_called_once_with(
-            "/project/1/key/aws", "PUT", None, expected_payload
-        )
-        assert response.type == "aws"
-
-
-class ProjectKeyTests:
-    def test_project_key_properties(self, project_key):
-        assert isinstance(project_key.type, str)
-        assert isinstance(project_key.key_id, str)
-        assert isinstance(project_key.project_uuid, str)
-
-    def test_project_key_str(self, project_key):
-        string = str(project_key)
-        regex = r"^{.*\"type\": \"" + project_key.type + r"\".*}$"
-        assert isinstance(string, str)
-        assert re.match(regex, string)
-
-    def test_project_key_repr(self, project_key):
-        string = repr(project_key)
-        regex = (
-            r"^ProjectKey\( proximl , \*\*{.*'type': '" + project_key.type + r"'.*}\)$"
-        )
-        assert isinstance(string, str)
-        assert re.match(regex, string)
-
-    def test_project_key_bool(self, project_key, mock_proximl):
-        empty_project_key = specimen.ProjectKey(mock_proximl)
-        assert bool(project_key)
-        assert not bool(empty_project_key)