proximl 0.5.9__tar.gz → 0.5.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. {proximl-0.5.9/proximl.egg-info → proximl-0.5.10}/PKG-INFO +1 -1
  2. {proximl-0.5.9 → proximl-0.5.10}/proximl/__init__.py +1 -1
  3. {proximl-0.5.9 → proximl-0.5.10}/proximl/checkpoints.py +14 -3
  4. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/datastore.py +2 -7
  5. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/data_connectors.py +8 -0
  6. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/datastores.py +9 -19
  7. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/nodes.py +45 -0
  8. {proximl-0.5.9 → proximl-0.5.10}/proximl/datasets.py +14 -3
  9. {proximl-0.5.9 → proximl-0.5.10}/proximl/jobs.py +2 -13
  10. {proximl-0.5.9 → proximl-0.5.10}/proximl/models.py +14 -3
  11. {proximl-0.5.9 → proximl-0.5.10}/proximl/volumes.py +15 -3
  12. {proximl-0.5.9 → proximl-0.5.10/proximl.egg-info}/PKG-INFO +1 -1
  13. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_checkpoints_integration.py +0 -1
  14. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_models_integration.py +0 -1
  15. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_datastores_unit.py +1 -5
  16. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_checkpoints_unit.py +15 -23
  17. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_datasets_unit.py +15 -20
  18. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_models_unit.py +13 -16
  19. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_volumes_unit.py +3 -0
  20. {proximl-0.5.9 → proximl-0.5.10}/LICENSE +0 -0
  21. {proximl-0.5.9 → proximl-0.5.10}/README.md +0 -0
  22. {proximl-0.5.9 → proximl-0.5.10}/examples/__init__.py +0 -0
  23. {proximl-0.5.9 → proximl-0.5.10}/examples/create_dataset_and_training_job.py +0 -0
  24. {proximl-0.5.9 → proximl-0.5.10}/examples/local_storage.py +0 -0
  25. {proximl-0.5.9 → proximl-0.5.10}/examples/training_inference_pipeline.py +0 -0
  26. {proximl-0.5.9 → proximl-0.5.10}/proximl/__main__.py +0 -0
  27. {proximl-0.5.9 → proximl-0.5.10}/proximl/auth.py +0 -0
  28. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/__init__.py +0 -0
  29. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/checkpoint.py +0 -0
  30. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/__init__.py +0 -0
  31. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/data_connector.py +0 -0
  32. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/device.py +0 -0
  33. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/node.py +0 -0
  34. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/provider.py +0 -0
  35. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/region.py +0 -0
  36. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/cloudbender/service.py +0 -0
  37. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/connection.py +0 -0
  38. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/dataset.py +0 -0
  39. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/environment.py +0 -0
  40. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/gpu.py +0 -0
  41. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/job/__init__.py +0 -0
  42. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/job/create.py +0 -0
  43. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/model.py +0 -0
  44. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/project/__init__.py +0 -0
  45. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/project/key.py +0 -0
  46. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/project/secret.py +0 -0
  47. {proximl-0.5.9 → proximl-0.5.10}/proximl/cli/volume.py +0 -0
  48. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/__init__.py +0 -0
  49. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/cloudbender.py +0 -0
  50. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/device_configs.py +0 -0
  51. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/devices.py +0 -0
  52. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/providers.py +0 -0
  53. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/regions.py +0 -0
  54. {proximl-0.5.9 → proximl-0.5.10}/proximl/cloudbender/services.py +0 -0
  55. {proximl-0.5.9 → proximl-0.5.10}/proximl/connections.py +0 -0
  56. {proximl-0.5.9 → proximl-0.5.10}/proximl/environments.py +0 -0
  57. {proximl-0.5.9 → proximl-0.5.10}/proximl/exceptions.py +0 -0
  58. {proximl-0.5.9 → proximl-0.5.10}/proximl/gpu_types.py +0 -0
  59. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/__init__.py +0 -0
  60. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/data_connectors.py +0 -0
  61. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/datastores.py +0 -0
  62. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/keys.py +0 -0
  63. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/projects.py +0 -0
  64. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/secrets.py +0 -0
  65. {proximl-0.5.9 → proximl-0.5.10}/proximl/projects/services.py +0 -0
  66. {proximl-0.5.9 → proximl-0.5.10}/proximl/proximl.py +0 -0
  67. {proximl-0.5.9 → proximl-0.5.10}/proximl.egg-info/SOURCES.txt +0 -0
  68. {proximl-0.5.9 → proximl-0.5.10}/proximl.egg-info/dependency_links.txt +0 -0
  69. {proximl-0.5.9 → proximl-0.5.10}/proximl.egg-info/entry_points.txt +0 -0
  70. {proximl-0.5.9 → proximl-0.5.10}/proximl.egg-info/requires.txt +0 -0
  71. {proximl-0.5.9 → proximl-0.5.10}/proximl.egg-info/top_level.txt +0 -0
  72. {proximl-0.5.9 → proximl-0.5.10}/pyproject.toml +0 -0
  73. {proximl-0.5.9 → proximl-0.5.10}/setup.cfg +0 -0
  74. {proximl-0.5.9 → proximl-0.5.10}/setup.py +0 -0
  75. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/__init__.py +0 -0
  76. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/cloudbender/__init__.py +0 -0
  77. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/cloudbender/test_providers_integration.py +0 -0
  78. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/conftest.py +0 -0
  79. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/projects/__init__.py +0 -0
  80. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/projects/conftest.py +0 -0
  81. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/projects/test_projects_integration.py +0 -0
  82. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/projects/test_projects_keys_integration.py +0 -0
  83. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/projects/test_projects_secrets_integration.py +0 -0
  84. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_datasets_integration.py +0 -0
  85. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_environments_integration.py +0 -0
  86. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_gpu_types_integration.py +0 -0
  87. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_jobs_integration.py +0 -0
  88. {proximl-0.5.9 → proximl-0.5.10}/tests/integration/test_volumes_integration.py +0 -0
  89. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/__init__.py +0 -0
  90. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/__init__.py +0 -0
  91. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/__init__.py +0 -0
  92. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/test_cli_datastore_unit.py +0 -0
  93. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/test_cli_device_unit.py +0 -0
  94. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/test_cli_node_unit.py +0 -0
  95. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/test_cli_provider_unit.py +0 -0
  96. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/test_cli_region_unit.py +0 -0
  97. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/cloudbender/test_cli_service_unit.py +0 -0
  98. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/conftest.py +0 -0
  99. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_checkpoint_unit.py +0 -0
  100. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_datasets_unit.py +0 -0
  101. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_environment_unit.py +0 -0
  102. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_gpu_unit.py +0 -0
  103. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_job_unit.py +0 -0
  104. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_model_unit.py +0 -0
  105. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_project_unit.py +0 -0
  106. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cli/test_cli_volume_unit.py +0 -0
  107. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/__init__.py +0 -0
  108. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_data_connectors_unit.py +0 -0
  109. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_device_configs_unit.py +0 -0
  110. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_devices_unit.py +0 -0
  111. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_nodes_unit.py +0 -0
  112. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_providers_unit.py +0 -0
  113. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_regions_unit.py +0 -0
  114. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/cloudbender/test_services_unit.py +0 -0
  115. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/conftest.py +0 -0
  116. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/__init__.py +0 -0
  117. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/test_project_data_connectors_unit.py +0 -0
  118. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/test_project_datastores_unit.py +0 -0
  119. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/test_project_keys_unit.py +0 -0
  120. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/test_project_secrets_unit.py +0 -0
  121. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/test_project_services_unit.py +0 -0
  122. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/projects/test_projects_unit.py +0 -0
  123. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_auth_unit.py +0 -0
  124. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_connections_unit.py +0 -0
  125. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_environments_unit.py +0 -0
  126. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_exceptions.py +0 -0
  127. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_gpu_types_unit.py +0 -0
  128. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_jobs_unit.py +0 -0
  129. {proximl-0.5.9 → proximl-0.5.10}/tests/unit/test_proximl_unit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: proximl
3
- Version: 0.5.9
3
+ Version: 0.5.10
4
4
  Summary: proxiML client SDK and command line utilities
5
5
  Home-page: https://github.com/proxiML/python-sdk
6
6
  Author: proxiML
@@ -13,5 +13,5 @@ logging.basicConfig(
13
13
  logger = logging.getLogger(__name__)
14
14
 
15
15
 
16
- __version__ = "0.5.9"
16
+ __version__ = "0.5.10"
17
17
  __all__ = "ProxiML"
@@ -31,13 +31,24 @@ class Checkpoints(object):
31
31
  datasets = [Checkpoint(self.proximl, **dataset) for dataset in resp]
32
32
  return datasets
33
33
 
34
- async def create(self, name, source_type, source_uri, **kwargs):
34
+ async def create(
35
+ self,
36
+ name,
37
+ source_type,
38
+ source_uri,
39
+ type="evefs",
40
+ project_uuid=None,
41
+ **kwargs,
42
+ ):
43
+ if not project_uuid:
44
+ project_uuid = self.proximl.active_project
35
45
  data = dict(
36
46
  name=name,
37
47
  source_type=source_type,
38
48
  source_uri=source_uri,
39
- source_options=kwargs.get("source_options"),
40
- project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
49
+ project_uuid=project_uuid,
50
+ type=type,
51
+ **kwargs,
41
52
  )
42
53
  payload = {k: v for k, v in data.items() if v is not None}
43
54
  logging.info(f"Creating Checkpoint {name}")
@@ -29,13 +29,11 @@ def datastore(config):
29
29
  def list(config, provider, region):
30
30
  """List datastores."""
31
31
  data = [
32
- ["ID", "NAME", "TYPE", "URI", "ROOT"],
32
+ ["ID", "NAME", "TYPE"],
33
33
  [
34
34
  "-" * 80,
35
35
  "-" * 80,
36
36
  "-" * 80,
37
- "-" * 80,
38
- "-" * 80,
39
37
  ],
40
38
  ]
41
39
 
@@ -51,15 +49,12 @@ def list(config, provider, region):
51
49
  datastore.id,
52
50
  datastore.name,
53
51
  datastore.type,
54
- datastore.uri,
55
- datastore.root,
56
52
  ]
57
53
  )
58
54
 
59
55
  for row in data:
60
56
  click.echo(
61
- "{: >37.36} {: >29.28} {: >9.8} {: >12.11} {: >12.11}"
62
- "".format(*row),
57
+ "{: >37.36} {: >29.28} {: >9.8} " "".format(*row),
63
58
  file=config.stdout,
64
59
  )
65
60
 
@@ -1,5 +1,13 @@
1
1
  import json
2
2
  import logging
3
+ import asyncio
4
+ import math
5
+
6
+ from proximl.exceptions import (
7
+ ApiError,
8
+ SpecificationError,
9
+ ProxiMLException,
10
+ )
3
11
 
4
12
 
5
13
  class DataConnectors(object):
@@ -1,5 +1,13 @@
1
1
  import json
2
2
  import logging
3
+ import asyncio
4
+ import math
5
+
6
+ from proximl.exceptions import (
7
+ ApiError,
8
+ SpecificationError,
9
+ ProxiMLException,
10
+ )
3
11
 
4
12
 
5
13
  class Datastores(object):
@@ -20,9 +28,7 @@ class Datastores(object):
20
28
  "GET",
21
29
  kwargs,
22
30
  )
23
- datastores = [
24
- Datastore(self.proximl, **datastore) for datastore in resp
25
- ]
31
+ datastores = [Datastore(self.proximl, **datastore) for datastore in resp]
26
32
  return datastores
27
33
 
28
34
  async def create(
@@ -31,18 +37,12 @@ class Datastores(object):
31
37
  region_uuid,
32
38
  name,
33
39
  type,
34
- uri,
35
- root,
36
- options=None,
37
40
  **kwargs,
38
41
  ):
39
42
  logging.info(f"Creating Datastore {name}")
40
43
  data = dict(
41
44
  name=name,
42
45
  type=type,
43
- uri=uri,
44
- root=root,
45
- options=options,
46
46
  **kwargs,
47
47
  )
48
48
  payload = {k: v for k, v in data.items() if v is not None}
@@ -73,8 +73,6 @@ class Datastore:
73
73
  self._region_uuid = self._datastore.get("region_uuid")
74
74
  self._type = self._datastore.get("type")
75
75
  self._name = self._datastore.get("name")
76
- self._uri = self._datastore.get("uri")
77
- self._root = self._datastore.get("root")
78
76
 
79
77
  @property
80
78
  def id(self) -> str:
@@ -96,14 +94,6 @@ class Datastore:
96
94
  def name(self) -> str:
97
95
  return self._name
98
96
 
99
- @property
100
- def uri(self) -> str:
101
- return self._uri
102
-
103
- @property
104
- def root(self) -> str:
105
- return self._root
106
-
107
97
  def __str__(self):
108
98
  return json.dumps({k: v for k, v in self._datastore.items()})
109
99
 
@@ -1,5 +1,13 @@
1
1
  import json
2
2
  import logging
3
+ import asyncio
4
+ import math
5
+
6
+ from proximl.exceptions import (
7
+ ApiError,
8
+ SpecificationError,
9
+ ProxiMLException,
10
+ )
3
11
 
4
12
 
5
13
  class Nodes(object):
@@ -153,3 +161,40 @@ class Node:
153
161
  None,
154
162
  dict(command=command),
155
163
  )
164
+
165
+ async def wait_for(self, status, timeout=300):
166
+ if self.status == status:
167
+ return
168
+ valid_statuses = ["active", "maintenance", "offline", "stopped", "archived"]
169
+ if not status in valid_statuses:
170
+ raise SpecificationError(
171
+ "status",
172
+ f"Invalid wait_for status {status}. Valid statuses are: {valid_statuses}",
173
+ )
174
+ MAX_TIMEOUT = 24 * 60 * 60
175
+ if timeout > MAX_TIMEOUT:
176
+ raise SpecificationError(
177
+ "timeout",
178
+ f"timeout must be less than {MAX_TIMEOUT} seconds.",
179
+ )
180
+
181
+ POLL_INTERVAL_MIN = 5
182
+ POLL_INTERVAL_MAX = 60
183
+ POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
184
+ retry_count = math.ceil(timeout / POLL_INTERVAL)
185
+ count = 0
186
+ while count < retry_count:
187
+ await asyncio.sleep(POLL_INTERVAL)
188
+ try:
189
+ await self.refresh()
190
+ except ApiError as e:
191
+ if status == "archived" and e.status == 404:
192
+ return
193
+ raise e
194
+ if self.status == status:
195
+ return self
196
+ else:
197
+ count += 1
198
+ logging.debug(f"self: {self}, retry count {count}")
199
+
200
+ raise ProxiMLException(f"Timeout waiting for {status}")
@@ -31,13 +31,24 @@ class Datasets(object):
31
31
  datasets = [Dataset(self.proximl, **dataset) for dataset in resp]
32
32
  return datasets
33
33
 
34
- async def create(self, name, source_type, source_uri, **kwargs):
34
+ async def create(
35
+ self,
36
+ name,
37
+ source_type,
38
+ source_uri,
39
+ type="evefs",
40
+ project_uuid=None,
41
+ **kwargs,
42
+ ):
43
+ if not project_uuid:
44
+ project_uuid = self.proximl.active_project
35
45
  data = dict(
36
46
  name=name,
37
47
  source_type=source_type,
38
48
  source_uri=source_uri,
39
- source_options=kwargs.get("source_options"),
40
- project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
49
+ project_uuid=project_uuid,
50
+ type=type,
51
+ **kwargs,
41
52
  )
42
53
  payload = {k: v for k, v in data.items() if v is not None}
43
54
  logging.info(f"Creating Dataset {name}")
@@ -476,7 +476,6 @@ class Job:
476
476
  return
477
477
  valid_statuses = [
478
478
  "waiting for data/model download",
479
- "waiting for GPUs",
480
479
  "waiting for resources",
481
480
  "running",
482
481
  "stopped",
@@ -488,11 +487,6 @@ class Job:
488
487
  "status",
489
488
  f"Invalid wait_for status {status}. Valid statuses are: {valid_statuses}",
490
489
  )
491
- if status == "waiting for GPUs":
492
- warnings.warn(
493
- "'waiting for GPUs' status is deprecated, use 'waiting for resources' instead.",
494
- DeprecationWarning,
495
- )
496
490
  if (self.type == "training") and status == "stopped":
497
491
  warnings.warn(
498
492
  "'stopped' status is deprecated for training jobs, use 'finished' instead.",
@@ -523,12 +517,8 @@ class Job:
523
517
  self.status == status
524
518
  or (
525
519
  status
526
- in [
527
- "waiting for GPUs",
528
- "waiting for resources",
529
- ] ## this status could be very short and the polling could miss it
530
- and self.status
531
- not in ["new", "waiting for GPUs", "waiting for resources"]
520
+ == "waiting for resources" ## this status could be very short and the polling could miss it
521
+ and self.status not in ["new", "waiting for resources"]
532
522
  )
533
523
  or (
534
524
  status
@@ -536,7 +526,6 @@ class Job:
536
526
  and self.status
537
527
  not in [
538
528
  "new",
539
- "waiting for GPUs",
540
529
  "waiting for resources",
541
530
  "waiting for data/model download",
542
531
  ]
@@ -26,13 +26,24 @@ class Models(object):
26
26
  models = [Model(self.proximl, **model) for model in resp]
27
27
  return models
28
28
 
29
- async def create(self, name, source_type, source_uri, **kwargs):
29
+ async def create(
30
+ self,
31
+ name,
32
+ source_type,
33
+ source_uri,
34
+ type="evefs",
35
+ project_uuid=None,
36
+ **kwargs,
37
+ ):
38
+ if not project_uuid:
39
+ project_uuid = self.proximl.active_project
30
40
  data = dict(
31
41
  name=name,
32
42
  source_type=source_type,
33
43
  source_uri=source_uri,
34
- source_options=kwargs.get("source_options"),
35
- project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
44
+ project_uuid=project_uuid,
45
+ type=type,
46
+ **kwargs,
36
47
  )
37
48
  payload = {k: v for k, v in data.items() if v is not None}
38
49
  logging.info(f"Creating Model {name}")
@@ -26,14 +26,26 @@ class Volumes(object):
26
26
  volumes = [Volume(self.proximl, **volume) for volume in resp]
27
27
  return volumes
28
28
 
29
- async def create(self, name, source_type, source_uri, capacity, **kwargs):
29
+ async def create(
30
+ self,
31
+ name,
32
+ source_type,
33
+ source_uri,
34
+ capacity,
35
+ type="evefs",
36
+ project_uuid=None,
37
+ **kwargs,
38
+ ):
39
+ if not project_uuid:
40
+ project_uuid = self.proximl.active_project
30
41
  data = dict(
31
42
  name=name,
32
43
  source_type=source_type,
33
44
  source_uri=source_uri,
34
45
  capacity=capacity,
35
- source_options=kwargs.get("source_options"),
36
- project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
46
+ project_uuid=project_uuid,
47
+ type=type,
48
+ **kwargs,
37
49
  )
38
50
  payload = {k: v for k, v in data.items() if v is not None}
39
51
  logging.info(f"Creating Volume {name}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: proximl
3
- Version: 0.5.9
3
+ Version: 0.5.10
4
4
  Summary: proxiML client SDK and command line utilities
5
5
  Home-page: https://github.com/proxiML/python-sdk
6
6
  Author: proxiML
@@ -59,7 +59,6 @@ async def test_checkpoint_wasabi(proximl, capsys):
59
59
  name="CLI Automated Wasabi",
60
60
  source_type="wasabi",
61
61
  source_uri="s3://proximl-example/models/proximl-examples",
62
- capacity="10G",
63
62
  source_options=dict(endpoint_url="https://s3.wasabisys.com"),
64
63
  )
65
64
  checkpoint = await checkpoint.wait_for("ready", 300)
@@ -55,7 +55,6 @@ async def test_model_wasabi(proximl, capsys):
55
55
  name="CLI Automated Wasabi",
56
56
  source_type="wasabi",
57
57
  source_uri="s3://proximl-example/models/proximl-examples",
58
- capacity="10G",
59
58
  source_options=dict(endpoint_url="https://s3.wasabisys.com"),
60
59
  )
61
60
  model = await model.wait_for("ready", 300)
@@ -119,8 +119,6 @@ class datastoreTests:
119
119
  assert isinstance(datastore.region_uuid, str)
120
120
  assert isinstance(datastore.type, str)
121
121
  assert isinstance(datastore.name, str)
122
- assert isinstance(datastore.uri, str)
123
- assert isinstance(datastore.root, str)
124
122
 
125
123
  def test_datastore_str(self, datastore):
126
124
  string = str(datastore)
@@ -131,9 +129,7 @@ class datastoreTests:
131
129
  def test_datastore_repr(self, datastore):
132
130
  string = repr(datastore)
133
131
  regex = (
134
- r"^Datastore\( proximl , \*\*{.*'store_id': '"
135
- + datastore.id
136
- + r"'.*}\)$"
132
+ r"^Datastore\( proximl , \*\*{.*'store_id': '" + datastore.id + r"'.*}\)$"
137
133
  )
138
134
  assert isinstance(string, str)
139
135
  assert re.match(regex, string)
@@ -44,9 +44,7 @@ class CheckpointsTests:
44
44
  api_response = dict()
45
45
  mock_proximl._query = AsyncMock(return_value=api_response)
46
46
  await checkpoints.get("1234")
47
- mock_proximl._query.assert_called_once_with(
48
- "/checkpoint/1234", "GET", dict()
49
- )
47
+ mock_proximl._query.assert_called_once_with("/checkpoint/1234", "GET", dict())
50
48
 
51
49
  @mark.asyncio
52
50
  async def test_list_checkpoints(
@@ -57,9 +55,7 @@ class CheckpointsTests:
57
55
  api_response = dict()
58
56
  mock_proximl._query = AsyncMock(return_value=api_response)
59
57
  await checkpoints.list()
60
- mock_proximl._query.assert_called_once_with(
61
- "/checkpoint", "GET", dict()
62
- )
58
+ mock_proximl._query.assert_called_once_with("/checkpoint", "GET", dict())
63
59
 
64
60
  @mark.asyncio
65
61
  async def test_remove_checkpoint(
@@ -86,11 +82,13 @@ class CheckpointsTests:
86
82
  name="new checkpoint",
87
83
  source_type="aws",
88
84
  source_uri="s3://proximl-examples/checkpoints/resnet50",
85
+ type="evefs",
89
86
  )
90
87
  api_response = {
91
88
  "project_uuid": "cus-id-1",
92
89
  "checkpoint_uuid": "checkpoint-id-1",
93
90
  "name": "new checkpoint",
91
+ "type": "evefs",
94
92
  "status": "new",
95
93
  "source_type": "aws",
96
94
  "source_uri": "s3://proximl-examples/checkpoints/resnet50",
@@ -135,7 +133,9 @@ class CheckpointTests:
135
133
 
136
134
  @mark.asyncio
137
135
  async def test_checkpoint_get_log_url(self, checkpoint, mock_proximl):
138
- api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
136
+ api_response = (
137
+ "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
138
+ )
139
139
  mock_proximl._query = AsyncMock(return_value=api_response)
140
140
  response = await checkpoint.get_log_url()
141
141
  mock_proximl._query.assert_called_once_with(
@@ -163,7 +163,9 @@ class CheckpointTests:
163
163
  async def test_checkpoint_get_connection_utility_url(
164
164
  self, checkpoint, mock_proximl
165
165
  ):
166
- api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
166
+ api_response = (
167
+ "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
168
+ )
167
169
  mock_proximl._query = AsyncMock(return_value=api_response)
168
170
  response = await checkpoint.get_connection_utility_url()
169
171
  mock_proximl._query.assert_called_once_with(
@@ -338,9 +340,7 @@ class CheckpointTests:
338
340
  assert response.id == "data-id-1"
339
341
 
340
342
  @mark.asyncio
341
- async def test_checkpoint_wait_for_successful(
342
- self, checkpoint, mock_proximl
343
- ):
343
+ async def test_checkpoint_wait_for_successful(self, checkpoint, mock_proximl):
344
344
  api_response = {
345
345
  "customer_uuid": "cus-id-1",
346
346
  "checkpoint_uuid": "data-id-1",
@@ -373,9 +373,7 @@ class CheckpointTests:
373
373
  mock_proximl._query.assert_not_called()
374
374
 
375
375
  @mark.asyncio
376
- async def test_checkpoint_wait_for_incorrect_status(
377
- self, checkpoint, mock_proximl
378
- ):
376
+ async def test_checkpoint_wait_for_incorrect_status(self, checkpoint, mock_proximl):
379
377
  api_response = None
380
378
  mock_proximl._query = AsyncMock(return_value=api_response)
381
379
  with raises(SpecificationError):
@@ -383,9 +381,7 @@ class CheckpointTests:
383
381
  mock_proximl._query.assert_not_called()
384
382
 
385
383
  @mark.asyncio
386
- async def test_checkpoint_wait_for_with_delay(
387
- self, checkpoint, mock_proximl
388
- ):
384
+ async def test_checkpoint_wait_for_with_delay(self, checkpoint, mock_proximl):
389
385
  api_response_initial = dict(
390
386
  checkpoint_uuid="1",
391
387
  name="first one",
@@ -441,9 +437,7 @@ class CheckpointTests:
441
437
  self, checkpoint, mock_proximl
442
438
  ):
443
439
  mock_proximl._query = AsyncMock(
444
- side_effect=ApiError(
445
- 404, dict(errorMessage="Checkpoint Not Found")
446
- )
440
+ side_effect=ApiError(404, dict(errorMessage="Checkpoint Not Found"))
447
441
  )
448
442
  await checkpoint.wait_for("archived")
449
443
  mock_proximl._query.assert_called()
@@ -453,9 +447,7 @@ class CheckpointTests:
453
447
  self, checkpoint, mock_proximl
454
448
  ):
455
449
  mock_proximl._query = AsyncMock(
456
- side_effect=ApiError(
457
- 404, dict(errorMessage="Checkpoint Not Found")
458
- )
450
+ side_effect=ApiError(404, dict(errorMessage="Checkpoint Not Found"))
459
451
  )
460
452
  with raises(ApiError):
461
453
  await checkpoint.wait_for("ready")
@@ -28,6 +28,7 @@ def dataset(mock_proximl):
28
28
  dataset_uuid="1",
29
29
  project_uuid="proj-id-1",
30
30
  name="first one",
31
+ type="evefs",
31
32
  status="downloading",
32
33
  size=100000,
33
34
  createdAt="2020-12-31T23:59:59.000Z",
@@ -103,6 +104,7 @@ class DatasetsTests:
103
104
  name="new dataset",
104
105
  source_type="aws",
105
106
  source_uri="s3://proximl-examples/data/cifar10",
107
+ type="evefs",
106
108
  )
107
109
  api_response = {
108
110
  "customer_uuid": "cus-id-1",
@@ -110,6 +112,7 @@ class DatasetsTests:
110
112
  "dataset_uuid": "data-id-1",
111
113
  "name": "new dataset",
112
114
  "status": "new",
115
+ "type": "evefs",
113
116
  "source_type": "aws",
114
117
  "source_uri": "s3://proximl-examples/data/cifar10",
115
118
  "createdAt": "2020-12-20T16:46:23.909Z",
@@ -139,9 +142,7 @@ class DatasetTests:
139
142
  def test_dataset_repr(self, dataset):
140
143
  string = repr(dataset)
141
144
  regex = (
142
- r"^Dataset\( proximl , \*\*{.*'dataset_uuid': '"
143
- + dataset.id
144
- + r"'.*}\)$"
145
+ r"^Dataset\( proximl , \*\*{.*'dataset_uuid': '" + dataset.id + r"'.*}\)$"
145
146
  )
146
147
  assert isinstance(string, str)
147
148
  assert re.match(regex, string)
@@ -153,7 +154,9 @@ class DatasetTests:
153
154
 
154
155
  @mark.asyncio
155
156
  async def test_dataset_get_log_url(self, dataset, mock_proximl):
156
- api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
157
+ api_response = (
158
+ "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
159
+ )
157
160
  mock_proximl._query = AsyncMock(return_value=api_response)
158
161
  response = await dataset.get_log_url()
159
162
  mock_proximl._query.assert_called_once_with(
@@ -178,10 +181,10 @@ class DatasetTests:
178
181
  assert response == api_response
179
182
 
180
183
  @mark.asyncio
181
- async def test_dataset_get_connection_utility_url(
182
- self, dataset, mock_proximl
183
- ):
184
- api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
184
+ async def test_dataset_get_connection_utility_url(self, dataset, mock_proximl):
185
+ api_response = (
186
+ "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
187
+ )
185
188
  mock_proximl._query = AsyncMock(return_value=api_response)
186
189
  response = await dataset.get_connection_utility_url()
187
190
  mock_proximl._query.assert_called_once_with(
@@ -388,9 +391,7 @@ class DatasetTests:
388
391
  mock_proximl._query.assert_not_called()
389
392
 
390
393
  @mark.asyncio
391
- async def test_dataset_wait_for_incorrect_status(
392
- self, dataset, mock_proximl
393
- ):
394
+ async def test_dataset_wait_for_incorrect_status(self, dataset, mock_proximl):
394
395
  api_response = None
395
396
  mock_proximl._query = AsyncMock(return_value=api_response)
396
397
  with raises(SpecificationError):
@@ -435,9 +436,7 @@ class DatasetTests:
435
436
  mock_proximl._query.assert_called()
436
437
 
437
438
  @mark.asyncio
438
- async def test_dataset_wait_for_dataset_failed(
439
- self, dataset, mock_proximl
440
- ):
439
+ async def test_dataset_wait_for_dataset_failed(self, dataset, mock_proximl):
441
440
  api_response = dict(
442
441
  dataset_uuid="1",
443
442
  name="first one",
@@ -450,9 +449,7 @@ class DatasetTests:
450
449
  mock_proximl._query.assert_called()
451
450
 
452
451
  @mark.asyncio
453
- async def test_dataset_wait_for_archived_succeeded(
454
- self, dataset, mock_proximl
455
- ):
452
+ async def test_dataset_wait_for_archived_succeeded(self, dataset, mock_proximl):
456
453
  mock_proximl._query = AsyncMock(
457
454
  side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
458
455
  )
@@ -460,9 +457,7 @@ class DatasetTests:
460
457
  mock_proximl._query.assert_called()
461
458
 
462
459
  @mark.asyncio
463
- async def test_dataset_wait_for_unexpected_api_error(
464
- self, dataset, mock_proximl
465
- ):
460
+ async def test_dataset_wait_for_unexpected_api_error(self, dataset, mock_proximl):
466
461
  mock_proximl._query = AsyncMock(
467
462
  side_effect=ApiError(404, dict(errorMessage="Dataset Not Found"))
468
463
  )
@@ -28,6 +28,7 @@ def model(mock_proximl):
28
28
  model_uuid="1",
29
29
  project_uuid="proj-id-1",
30
30
  name="first one",
31
+ type="evefs",
31
32
  status="downloading",
32
33
  size=100000,
33
34
  createdAt="2020-12-31T23:59:59.000Z",
@@ -44,9 +45,7 @@ class ModelsTests:
44
45
  api_response = dict()
45
46
  mock_proximl._query = AsyncMock(return_value=api_response)
46
47
  await models.get("1234")
47
- mock_proximl._query.assert_called_once_with(
48
- "/model/1234", "GET", dict()
49
- )
48
+ mock_proximl._query.assert_called_once_with("/model/1234", "GET", dict())
50
49
 
51
50
  @mark.asyncio
52
51
  async def test_list_models(
@@ -84,12 +83,14 @@ class ModelsTests:
84
83
  name="new model",
85
84
  source_type="aws",
86
85
  source_uri="s3://proximl-examples/models/resnet50",
86
+ type="evefs",
87
87
  )
88
88
  api_response = {
89
89
  "customer_uuid": "cus-id-1",
90
90
  "model_uuid": "model-id-1",
91
91
  "name": "new model",
92
92
  "status": "new",
93
+ "type": "evefs",
93
94
  "source_type": "aws",
94
95
  "source_uri": "s3://proximl-examples/models/resnet50",
95
96
  "createdAt": "2020-12-20T16:46:23.909Z",
@@ -118,11 +119,7 @@ class ModelTests:
118
119
 
119
120
  def test_model_repr(self, model):
120
121
  string = repr(model)
121
- regex = (
122
- r"^Model\( proximl , \*\*{.*'model_uuid': '"
123
- + model.id
124
- + r"'.*}\)$"
125
- )
122
+ regex = r"^Model\( proximl , \*\*{.*'model_uuid': '" + model.id + r"'.*}\)$"
126
123
  assert isinstance(string, str)
127
124
  assert re.match(regex, string)
128
125
 
@@ -133,7 +130,9 @@ class ModelTests:
133
130
 
134
131
  @mark.asyncio
135
132
  async def test_model_get_log_url(self, model, mock_proximl):
136
- api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
133
+ api_response = (
134
+ "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
135
+ )
137
136
  mock_proximl._query = AsyncMock(return_value=api_response)
138
137
  response = await model.get_log_url()
139
138
  mock_proximl._query.assert_called_once_with(
@@ -159,7 +158,9 @@ class ModelTests:
159
158
 
160
159
  @mark.asyncio
161
160
  async def test_model_get_connection_utility_url(self, model, mock_proximl):
162
- api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
161
+ api_response = (
162
+ "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
163
+ )
163
164
  mock_proximl._query = AsyncMock(return_value=api_response)
164
165
  response = await model.get_connection_utility_url()
165
166
  mock_proximl._query.assert_called_once_with(
@@ -425,9 +426,7 @@ class ModelTests:
425
426
  mock_proximl._query.assert_called()
426
427
 
427
428
  @mark.asyncio
428
- async def test_model_wait_for_archived_succeeded(
429
- self, model, mock_proximl
430
- ):
429
+ async def test_model_wait_for_archived_succeeded(self, model, mock_proximl):
431
430
  mock_proximl._query = AsyncMock(
432
431
  side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
433
432
  )
@@ -435,9 +434,7 @@ class ModelTests:
435
434
  mock_proximl._query.assert_called()
436
435
 
437
436
  @mark.asyncio
438
- async def test_model_wait_for_unexpected_api_error(
439
- self, model, mock_proximl
440
- ):
437
+ async def test_model_wait_for_unexpected_api_error(self, model, mock_proximl):
441
438
  mock_proximl._query = AsyncMock(
442
439
  side_effect=ApiError(404, dict(errorMessage="Model Not Found"))
443
440
  )
@@ -26,6 +26,7 @@ def volume(mock_proximl):
26
26
  yield specimen.Volume(
27
27
  mock_proximl,
28
28
  id="1",
29
+ type="evefs",
29
30
  project_uuid="proj-id-1",
30
31
  name="first one",
31
32
  status="downloading",
@@ -86,12 +87,14 @@ class VolumesTests:
86
87
  source_type="aws",
87
88
  source_uri="s3://proximl-examples/volumes/resnet50",
88
89
  capacity="10G",
90
+ type="evefs",
89
91
  )
90
92
  api_response = {
91
93
  "project_uuid": "cus-id-1",
92
94
  "id": "volume-id-1",
93
95
  "name": "new volume",
94
96
  "status": "new",
97
+ "type": "evefs",
95
98
  "source_type": "aws",
96
99
  "capacity": "10G",
97
100
  "source_uri": "s3://proximl-examples/volumes/resnet50",
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes