proximl 0.5.17__py3-none-any.whl → 1.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. examples/local_storage.py +0 -2
  2. proximl/__init__.py +1 -1
  3. proximl/checkpoints.py +56 -57
  4. proximl/cli/__init__.py +6 -3
  5. proximl/cli/checkpoint.py +18 -57
  6. proximl/cli/dataset.py +17 -57
  7. proximl/cli/job/__init__.py +89 -67
  8. proximl/cli/job/create.py +51 -24
  9. proximl/cli/model.py +14 -56
  10. proximl/cli/volume.py +18 -57
  11. proximl/datasets.py +50 -55
  12. proximl/jobs.py +269 -69
  13. proximl/models.py +51 -55
  14. proximl/proximl.py +159 -114
  15. proximl/utils/__init__.py +1 -0
  16. proximl/{auth.py → utils/auth.py} +4 -3
  17. proximl/utils/transfer.py +647 -0
  18. proximl/volumes.py +48 -53
  19. {proximl-0.5.17.dist-info → proximl-1.0.1.dist-info}/METADATA +3 -3
  20. {proximl-0.5.17.dist-info → proximl-1.0.1.dist-info}/RECORD +52 -50
  21. tests/integration/test_checkpoints_integration.py +4 -3
  22. tests/integration/test_datasets_integration.py +5 -3
  23. tests/integration/test_jobs_integration.py +33 -27
  24. tests/integration/test_models_integration.py +7 -3
  25. tests/integration/test_volumes_integration.py +2 -2
  26. tests/unit/cli/test_cli_checkpoint_unit.py +312 -1
  27. tests/unit/cloudbender/test_nodes_unit.py +112 -0
  28. tests/unit/cloudbender/test_providers_unit.py +96 -0
  29. tests/unit/cloudbender/test_regions_unit.py +106 -0
  30. tests/unit/cloudbender/test_services_unit.py +141 -0
  31. tests/unit/conftest.py +23 -10
  32. tests/unit/projects/test_project_data_connectors_unit.py +39 -0
  33. tests/unit/projects/test_project_datastores_unit.py +37 -0
  34. tests/unit/projects/test_project_members_unit.py +46 -0
  35. tests/unit/projects/test_project_services_unit.py +65 -0
  36. tests/unit/projects/test_projects_unit.py +16 -0
  37. tests/unit/test_auth_unit.py +17 -2
  38. tests/unit/test_checkpoints_unit.py +256 -71
  39. tests/unit/test_datasets_unit.py +218 -68
  40. tests/unit/test_exceptions.py +133 -0
  41. tests/unit/test_gpu_types_unit.py +11 -1
  42. tests/unit/test_jobs_unit.py +1014 -95
  43. tests/unit/test_main_unit.py +20 -0
  44. tests/unit/test_models_unit.py +218 -70
  45. tests/unit/test_proximl_unit.py +627 -3
  46. tests/unit/test_volumes_unit.py +211 -70
  47. tests/unit/utils/__init__.py +1 -0
  48. tests/unit/utils/test_transfer_unit.py +4260 -0
  49. proximl/cli/connection.py +0 -61
  50. proximl/connections.py +0 -621
  51. tests/unit/test_connections_unit.py +0 -182
  52. {proximl-0.5.17.dist-info → proximl-1.0.1.dist-info}/LICENSE +0 -0
  53. {proximl-0.5.17.dist-info → proximl-1.0.1.dist-info}/WHEEL +0 -0
  54. {proximl-0.5.17.dist-info → proximl-1.0.1.dist-info}/entry_points.txt +0 -0
  55. {proximl-0.5.17.dist-info → proximl-1.0.1.dist-info}/top_level.txt +0 -0
@@ -47,7 +47,9 @@ class VolumesTests:
47
47
  api_response = dict()
48
48
  mock_proximl._query = AsyncMock(return_value=api_response)
49
49
  await volumes.get("1234")
50
- mock_proximl._query.assert_called_once_with("/volume/1234", "GET", dict())
50
+ mock_proximl._query.assert_called_once_with(
51
+ "/volume/1234", "GET", dict()
52
+ )
51
53
 
52
54
  @mark.asyncio
53
55
  async def test_list_volumes(
@@ -137,9 +139,7 @@ class VolumeTests:
137
139
 
138
140
  @mark.asyncio
139
141
  async def test_volume_get_log_url(self, volume, mock_proximl):
140
- api_response = (
141
- "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
142
- )
142
+ api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
143
143
  mock_proximl._query = AsyncMock(return_value=api_response)
144
144
  response = await volume.get_log_url()
145
145
  mock_proximl._query.assert_called_once_with(
@@ -164,79 +164,73 @@ class VolumeTests:
164
164
  assert response == api_response
165
165
 
166
166
  @mark.asyncio
167
- async def test_volume_get_connection_utility_url(self, volume, mock_proximl):
168
- api_response = (
169
- "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/vpn/first_one.zip"
170
- )
171
- mock_proximl._query = AsyncMock(return_value=api_response)
172
- response = await volume.get_connection_utility_url()
173
- mock_proximl._query.assert_called_once_with(
174
- "/volume/1/download", "GET", dict(project_uuid="proj-id-1")
167
+ async def test_volume_connect_downloading_status(self, mock_proximl):
168
+ volume = specimen.Volume(
169
+ mock_proximl,
170
+ id="1",
171
+ project_uuid="proj-id-1",
172
+ name="test volume",
173
+ status="downloading",
174
+ auth_token="test-token",
175
+ hostname="example.com",
176
+ source_uri="/path/to/source",
175
177
  )
176
- assert response == api_response
177
178
 
178
- def test_volume_get_connection_details_no_vpn(self, volume):
179
- details = volume.get_connection_details()
180
- expected_details = dict()
181
- assert details == expected_details
179
+ with patch(
180
+ "proximl.volumes.Volume.refresh", new_callable=AsyncMock
181
+ ) as mock_refresh:
182
+ with patch(
183
+ "proximl.volumes.upload", new_callable=AsyncMock
184
+ ) as mock_upload:
185
+ await volume.connect()
186
+ mock_refresh.assert_called_once()
187
+ mock_upload.assert_called_once_with(
188
+ "example.com", "test-token", "/path/to/source"
189
+ )
182
190
 
183
- def test_volume_get_connection_details_local_data(self, mock_proximl):
191
+ @mark.asyncio
192
+ async def test_volume_connect_exporting_status(
193
+ self, mock_proximl, tmp_path
194
+ ):
195
+ output_dir = str(tmp_path / "output")
184
196
  volume = specimen.Volume(
185
197
  mock_proximl,
186
198
  id="1",
187
- project_uuid="a",
188
- name="first one",
189
- status="new",
190
- capacity="10G",
191
- createdAt="2020-12-31T23:59:59.000Z",
192
- source_type="local",
193
- source_uri="~/tensorflow-example",
194
- vpn={
195
- "status": "new",
196
- "cidr": "10.106.171.0/24",
197
- "client": {
198
- "port": "36017",
199
- "id": "cus-id-1",
200
- "address": "10.106.171.253",
201
- "ssh_port": 46600,
202
- },
203
- "net_prefix_type_id": 1,
204
- },
205
- )
206
- details = volume.get_connection_details()
207
- expected_details = dict(
208
- project_uuid="a",
209
- entity_type="volume",
210
- cidr="10.106.171.0/24",
211
- ssh_port=46600,
212
- input_path="~/tensorflow-example",
213
- output_path=None,
214
- )
215
- assert details == expected_details
199
+ project_uuid="proj-id-1",
200
+ name="test volume",
201
+ status="exporting",
202
+ auth_token="test-token",
203
+ hostname="example.com",
204
+ output_uri=output_dir,
205
+ )
216
206
 
217
- @mark.asyncio
218
- async def test_volume_connect(self, volume, mock_proximl):
219
207
  with patch(
220
- "proximl.volumes.Connection",
221
- autospec=True,
222
- ) as mock_connection:
223
- connection = mock_connection.return_value
224
- connection.status = "connected"
225
- resp = await volume.connect()
226
- connection.start.assert_called_once()
227
- assert resp == "connected"
208
+ "proximl.volumes.Volume.refresh", new_callable=AsyncMock
209
+ ) as mock_refresh:
210
+ with patch(
211
+ "proximl.volumes.download", new_callable=AsyncMock
212
+ ) as mock_download:
213
+ await volume.connect()
214
+ mock_refresh.assert_called_once()
215
+ mock_download.assert_called_once_with(
216
+ "example.com", "test-token", output_dir
217
+ )
228
218
 
229
219
  @mark.asyncio
230
- async def test_volume_disconnect(self, volume, mock_proximl):
231
- with patch(
232
- "proximl.volumes.Connection",
233
- autospec=True,
234
- ) as mock_connection:
235
- connection = mock_connection.return_value
236
- connection.status = "removed"
237
- resp = await volume.disconnect()
238
- connection.stop.assert_called_once()
239
- assert resp == "removed"
220
+ async def test_volume_connect_invalid_status(self, mock_proximl):
221
+ volume = specimen.Volume(
222
+ mock_proximl,
223
+ id="1",
224
+ project_uuid="proj-id-1",
225
+ name="test volume",
226
+ status="ready",
227
+ )
228
+
229
+ with raises(
230
+ SpecificationError,
231
+ match="You can only connect to downloading or exporting volumes",
232
+ ):
233
+ await volume.connect()
240
234
 
241
235
  @mark.asyncio
242
236
  async def test_volume_remove(self, volume, mock_proximl):
@@ -375,7 +369,9 @@ class VolumeTests:
375
369
  mock_proximl._query.assert_not_called()
376
370
 
377
371
  @mark.asyncio
378
- async def test_volume_wait_for_incorrect_status(self, volume, mock_proximl):
372
+ async def test_volume_wait_for_incorrect_status(
373
+ self, volume, mock_proximl
374
+ ):
379
375
  api_response = None
380
376
  mock_proximl._query = AsyncMock(return_value=api_response)
381
377
  with raises(SpecificationError):
@@ -433,7 +429,9 @@ class VolumeTests:
433
429
  mock_proximl._query.assert_called()
434
430
 
435
431
  @mark.asyncio
436
- async def test_volume_wait_for_archived_succeeded(self, volume, mock_proximl):
432
+ async def test_volume_wait_for_archived_succeeded(
433
+ self, volume, mock_proximl
434
+ ):
437
435
  mock_proximl._query = AsyncMock(
438
436
  side_effect=ApiError(404, dict(errorMessage="Volume Not Found"))
439
437
  )
@@ -441,10 +439,153 @@ class VolumeTests:
441
439
  mock_proximl._query.assert_called()
442
440
 
443
441
  @mark.asyncio
444
- async def test_volume_wait_for_unexpected_api_error(self, volume, mock_proximl):
442
+ async def test_volume_wait_for_unexpected_api_error(
443
+ self, volume, mock_proximl
444
+ ):
445
445
  mock_proximl._query = AsyncMock(
446
446
  side_effect=ApiError(404, dict(errorMessage="Volume Not Found"))
447
447
  )
448
448
  with raises(ApiError):
449
449
  await volume.wait_for("ready")
450
450
  mock_proximl._query.assert_called()
451
+
452
+ @mark.asyncio
453
+ async def test_volume_rename(self, volume, mock_proximl):
454
+ api_response = dict(
455
+ id="1",
456
+ name="renamed volume",
457
+ project_uuid="proj-id-1",
458
+ status="ready",
459
+ )
460
+ mock_proximl._query = AsyncMock(return_value=api_response)
461
+ result = await volume.rename("renamed volume")
462
+ mock_proximl._query.assert_called_once_with(
463
+ "/volume/1",
464
+ "PATCH",
465
+ dict(project_uuid="proj-id-1"),
466
+ dict(name="renamed volume"),
467
+ )
468
+ assert result == volume
469
+ assert volume.name == "renamed volume"
470
+
471
+ @mark.asyncio
472
+ async def test_volume_export(self, volume, mock_proximl):
473
+ api_response = dict(
474
+ id="1",
475
+ name="first one",
476
+ project_uuid="proj-id-1",
477
+ status="exporting",
478
+ )
479
+ mock_proximl._query = AsyncMock(return_value=api_response)
480
+ result = await volume.export("aws", "s3://bucket/path", dict(key="value"))
481
+ mock_proximl._query.assert_called_once_with(
482
+ "/volume/1/export",
483
+ "POST",
484
+ dict(project_uuid="proj-id-1"),
485
+ dict(
486
+ output_type="aws",
487
+ output_uri="s3://bucket/path",
488
+ output_options=dict(key="value"),
489
+ ),
490
+ )
491
+ assert result == volume
492
+ assert volume.status == "exporting"
493
+
494
+ @mark.asyncio
495
+ async def test_volume_export_default_options(self, volume, mock_proximl):
496
+ api_response = dict(
497
+ id="1",
498
+ name="first one",
499
+ project_uuid="proj-id-1",
500
+ status="exporting",
501
+ )
502
+ mock_proximl._query = AsyncMock(return_value=api_response)
503
+ result = await volume.export("aws", "s3://bucket/path")
504
+ mock_proximl._query.assert_called_once_with(
505
+ "/volume/1/export",
506
+ "POST",
507
+ dict(project_uuid="proj-id-1"),
508
+ dict(
509
+ output_type="aws",
510
+ output_uri="s3://bucket/path",
511
+ output_options=dict(),
512
+ ),
513
+ )
514
+ assert result == volume
515
+
516
+ @mark.asyncio
517
+ async def test_volume_wait_for_timeout_validation(
518
+ self, volume, mock_proximl
519
+ ):
520
+ with raises(SpecificationError) as exc_info:
521
+ await volume.wait_for("ready", timeout=25 * 60 * 60) # > 24 hours
522
+ assert "timeout" in str(exc_info.value.attribute).lower()
523
+ assert "less than" in str(exc_info.value.message).lower()
524
+
525
+ @mark.asyncio
526
+ async def test_volume_connect_new_status_waits_for_downloading(
527
+ self, volume, mock_proximl
528
+ ):
529
+ """Test that connect() waits for downloading status when status is 'new'."""
530
+ volume._volume["status"] = "new"
531
+ volume._status = "new"
532
+ api_response_new = dict(
533
+ id="1",
534
+ name="first one",
535
+ status="new",
536
+ )
537
+ api_response_downloading = dict(
538
+ id="1",
539
+ name="first one",
540
+ status="downloading",
541
+ auth_token="token",
542
+ hostname="host",
543
+ source_uri="s3://bucket/path",
544
+ )
545
+ # wait_for calls refresh multiple times, then connect calls refresh again
546
+ mock_proximl._query = AsyncMock(
547
+ side_effect=[
548
+ api_response_new, # wait_for refresh 1
549
+ api_response_downloading, # wait_for refresh 2 (status matches, wait_for returns)
550
+ api_response_downloading, # connect refresh
551
+ ]
552
+ )
553
+ with patch("proximl.volumes.upload", new_callable=AsyncMock) as mock_upload:
554
+ await volume.connect()
555
+ # After refresh, status should be downloading
556
+ assert volume.status == "downloading"
557
+ mock_upload.assert_called_once()
558
+
559
+ @mark.asyncio
560
+ async def test_volume_connect_downloading_missing_properties(
561
+ self, volume, mock_proximl
562
+ ):
563
+ """Test connect() raises error when downloading status missing properties."""
564
+ volume._volume["status"] = "downloading"
565
+ api_response = dict(
566
+ id="1",
567
+ name="first one",
568
+ status="downloading",
569
+ # Missing auth_token, hostname, or source_uri
570
+ )
571
+ mock_proximl._query = AsyncMock(return_value=api_response)
572
+ with raises(SpecificationError) as exc_info:
573
+ await volume.connect()
574
+ assert "missing required connection properties" in str(exc_info.value.message).lower()
575
+
576
+ @mark.asyncio
577
+ async def test_volume_connect_exporting_missing_properties(
578
+ self, volume, mock_proximl
579
+ ):
580
+ """Test connect() raises error when exporting status missing properties."""
581
+ volume._volume["status"] = "exporting"
582
+ api_response = dict(
583
+ id="1",
584
+ name="first one",
585
+ status="exporting",
586
+ # Missing auth_token, hostname, or output_uri
587
+ )
588
+ mock_proximl._query = AsyncMock(return_value=api_response)
589
+ with raises(SpecificationError) as exc_info:
590
+ await volume.connect()
591
+ assert "missing required connection properties" in str(exc_info.value.message).lower()
@@ -0,0 +1 @@
1
+ """Unit tests for utility modules."""