datacrunch 1.14.0__py3-none-any.whl → 1.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacrunch/InferenceClient/inference_client.py +200 -65
- datacrunch/__init__.py +2 -0
- datacrunch/_version.py +6 -0
- datacrunch/authentication/authentication.py +7 -14
- datacrunch/balance/balance.py +1 -3
- datacrunch/constants.py +19 -17
- datacrunch/containers/containers.py +151 -123
- datacrunch/datacrunch.py +18 -18
- datacrunch/helpers.py +7 -2
- datacrunch/http_client/http_client.py +14 -14
- datacrunch/images/images.py +9 -3
- datacrunch/instance_types/instance_types.py +42 -35
- datacrunch/instances/instances.py +74 -53
- datacrunch/locations/locations.py +1 -2
- datacrunch/ssh_keys/ssh_keys.py +3 -4
- datacrunch/startup_scripts/startup_scripts.py +10 -8
- datacrunch/volume_types/volume_types.py +10 -8
- datacrunch/volumes/volumes.py +60 -73
- {datacrunch-1.14.0.dist-info → datacrunch-1.16.0.dist-info}/METADATA +46 -72
- datacrunch-1.16.0.dist-info/RECORD +35 -0
- datacrunch-1.16.0.dist-info/WHEEL +4 -0
- datacrunch/__version__.py +0 -1
- datacrunch-1.14.0.dist-info/RECORD +0 -69
- datacrunch-1.14.0.dist-info/WHEEL +0 -5
- datacrunch-1.14.0.dist-info/licenses/LICENSE +0 -21
- datacrunch-1.14.0.dist-info/top_level.txt +0 -2
- tests/__init__.py +0 -0
- tests/integration_tests/__init__.py +0 -0
- tests/integration_tests/conftest.py +0 -20
- tests/integration_tests/test_instances.py +0 -36
- tests/integration_tests/test_locations.py +0 -65
- tests/integration_tests/test_volumes.py +0 -94
- tests/unit_tests/__init__.py +0 -0
- tests/unit_tests/authentication/__init__.py +0 -0
- tests/unit_tests/authentication/test_authentication.py +0 -202
- tests/unit_tests/balance/__init__.py +0 -0
- tests/unit_tests/balance/test_balance.py +0 -25
- tests/unit_tests/conftest.py +0 -21
- tests/unit_tests/containers/__init__.py +0 -1
- tests/unit_tests/containers/test_containers.py +0 -959
- tests/unit_tests/http_client/__init__.py +0 -0
- tests/unit_tests/http_client/test_http_client.py +0 -193
- tests/unit_tests/images/__init__.py +0 -0
- tests/unit_tests/images/test_images.py +0 -41
- tests/unit_tests/instance_types/__init__.py +0 -0
- tests/unit_tests/instance_types/test_instance_types.py +0 -87
- tests/unit_tests/instances/__init__.py +0 -0
- tests/unit_tests/instances/test_instances.py +0 -483
- tests/unit_tests/ssh_keys/__init__.py +0 -0
- tests/unit_tests/ssh_keys/test_ssh_keys.py +0 -198
- tests/unit_tests/startup_scripts/__init__.py +0 -0
- tests/unit_tests/startup_scripts/test_startup_scripts.py +0 -196
- tests/unit_tests/test_datacrunch.py +0 -65
- tests/unit_tests/test_exceptions.py +0 -33
- tests/unit_tests/volume_types/__init__.py +0 -0
- tests/unit_tests/volume_types/test_volume_types.py +0 -50
- tests/unit_tests/volumes/__init__.py +0 -0
- tests/unit_tests/volumes/test_volumes.py +0 -641
|
@@ -26,47 +26,47 @@ FILESET_SECRETS_ENDPOINT = '/file-secrets'
|
|
|
26
26
|
class EnvVarType(str, Enum):
|
|
27
27
|
"""Types of environment variables that can be set in containers."""
|
|
28
28
|
|
|
29
|
-
PLAIN =
|
|
30
|
-
SECRET =
|
|
29
|
+
PLAIN = 'plain'
|
|
30
|
+
SECRET = 'secret'
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class SecretType(str, Enum):
|
|
34
34
|
"""Types of secrets that can be set in containers."""
|
|
35
35
|
|
|
36
|
-
GENERIC =
|
|
37
|
-
FILESET =
|
|
36
|
+
GENERIC = 'generic' # Regular secret, can be used in env vars
|
|
37
|
+
FILESET = 'file-secret' # A file secret that can be mounted into the container
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
class VolumeMountType(str, Enum):
|
|
41
41
|
"""Types of volume mounts that can be configured for containers."""
|
|
42
42
|
|
|
43
|
-
SCRATCH =
|
|
44
|
-
SECRET =
|
|
45
|
-
MEMORY =
|
|
46
|
-
SHARED =
|
|
43
|
+
SCRATCH = 'scratch'
|
|
44
|
+
SECRET = 'secret'
|
|
45
|
+
MEMORY = 'memory'
|
|
46
|
+
SHARED = 'shared'
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
class ContainerRegistryType(str, Enum):
|
|
50
50
|
"""Supported container registry types."""
|
|
51
51
|
|
|
52
|
-
GCR =
|
|
53
|
-
DOCKERHUB =
|
|
54
|
-
GITHUB =
|
|
55
|
-
AWS_ECR =
|
|
56
|
-
CUSTOM =
|
|
52
|
+
GCR = 'gcr'
|
|
53
|
+
DOCKERHUB = 'dockerhub'
|
|
54
|
+
GITHUB = 'ghcr'
|
|
55
|
+
AWS_ECR = 'aws-ecr'
|
|
56
|
+
CUSTOM = 'custom'
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
class ContainerDeploymentStatus(str, Enum):
|
|
60
60
|
"""Possible states of a container deployment."""
|
|
61
61
|
|
|
62
|
-
INITIALIZING =
|
|
63
|
-
HEALTHY =
|
|
64
|
-
DEGRADED =
|
|
65
|
-
UNHEALTHY =
|
|
66
|
-
PAUSED =
|
|
67
|
-
QUOTA_REACHED =
|
|
68
|
-
IMAGE_PULLING =
|
|
69
|
-
VERSION_UPDATING =
|
|
62
|
+
INITIALIZING = 'initializing'
|
|
63
|
+
HEALTHY = 'healthy'
|
|
64
|
+
DEGRADED = 'degraded'
|
|
65
|
+
UNHEALTHY = 'unhealthy'
|
|
66
|
+
PAUSED = 'paused'
|
|
67
|
+
QUOTA_REACHED = 'quota_reached'
|
|
68
|
+
IMAGE_PULLING = 'image_pulling'
|
|
69
|
+
VERSION_UPDATING = 'version_updating'
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
@dataclass_json
|
|
@@ -137,8 +137,7 @@ class VolumeMount:
|
|
|
137
137
|
@dataclass_json(undefined=Undefined.EXCLUDE)
|
|
138
138
|
@dataclass
|
|
139
139
|
class GeneralStorageMount(VolumeMount):
|
|
140
|
-
"""General storage volume mount configuration.
|
|
141
|
-
"""
|
|
140
|
+
"""General storage volume mount configuration."""
|
|
142
141
|
|
|
143
142
|
def __init__(self, mount_path: str):
|
|
144
143
|
"""Initialize a general scratch volume mount.
|
|
@@ -369,7 +368,8 @@ class Deployment:
|
|
|
369
368
|
containers: List[Container]
|
|
370
369
|
compute: ComputeResource
|
|
371
370
|
container_registry_settings: ContainerRegistrySettings = field(
|
|
372
|
-
default_factory=lambda: ContainerRegistrySettings(is_private=False)
|
|
371
|
+
default_factory=lambda: ContainerRegistrySettings(is_private=False)
|
|
372
|
+
)
|
|
373
373
|
is_spot: bool = False
|
|
374
374
|
endpoint_base_url: Optional[str] = None
|
|
375
375
|
scaling: Optional[ScalingOptions] = None
|
|
@@ -384,11 +384,10 @@ class Deployment:
|
|
|
384
384
|
str: A formatted string representation of the deployment.
|
|
385
385
|
"""
|
|
386
386
|
# Get all attributes except _inference_client
|
|
387
|
-
attrs = {k: v for k, v in self.__dict__.items() if k !=
|
|
388
|
-
'_inference_client'}
|
|
387
|
+
attrs = {k: v for k, v in self.__dict__.items() if k != '_inference_client'}
|
|
389
388
|
# Format each attribute
|
|
390
|
-
attr_strs = [f
|
|
391
|
-
return f
|
|
389
|
+
attr_strs = [f'{k}={repr(v)}' for k, v in attrs.items()]
|
|
390
|
+
return f'Deployment({", ".join(attr_strs)})'
|
|
392
391
|
|
|
393
392
|
def __repr__(self):
|
|
394
393
|
"""Returns a repr representation of the deployment, excluding sensitive information.
|
|
@@ -399,7 +398,9 @@ class Deployment:
|
|
|
399
398
|
return self.__str__()
|
|
400
399
|
|
|
401
400
|
@classmethod
|
|
402
|
-
def from_dict_with_inference_key(
|
|
401
|
+
def from_dict_with_inference_key(
|
|
402
|
+
cls, data: Dict[str, Any], inference_key: str = None
|
|
403
|
+
) -> 'Deployment':
|
|
403
404
|
"""Creates a Deployment instance from a dictionary with an inference key.
|
|
404
405
|
|
|
405
406
|
Args:
|
|
@@ -413,7 +414,7 @@ class Deployment:
|
|
|
413
414
|
if inference_key and deployment.endpoint_base_url:
|
|
414
415
|
deployment._inference_client = InferenceClient(
|
|
415
416
|
inference_key=inference_key,
|
|
416
|
-
endpoint_base_url=deployment.endpoint_base_url
|
|
417
|
+
endpoint_base_url=deployment.endpoint_base_url,
|
|
417
418
|
)
|
|
418
419
|
return deployment
|
|
419
420
|
|
|
@@ -427,11 +428,9 @@ class Deployment:
|
|
|
427
428
|
ValueError: If endpoint_base_url is not set.
|
|
428
429
|
"""
|
|
429
430
|
if self.endpoint_base_url is None:
|
|
430
|
-
raise ValueError(
|
|
431
|
-
"Endpoint base URL must be set to use inference client")
|
|
431
|
+
raise ValueError('Endpoint base URL must be set to use inference client')
|
|
432
432
|
self._inference_client = InferenceClient(
|
|
433
|
-
inference_key=inference_key,
|
|
434
|
-
endpoint_base_url=self.endpoint_base_url
|
|
433
|
+
inference_key=inference_key, endpoint_base_url=self.endpoint_base_url
|
|
435
434
|
)
|
|
436
435
|
|
|
437
436
|
def _validate_inference_client(self) -> None:
|
|
@@ -442,9 +441,18 @@ class Deployment:
|
|
|
442
441
|
"""
|
|
443
442
|
if self._inference_client is None:
|
|
444
443
|
raise ValueError(
|
|
445
|
-
|
|
444
|
+
'Inference client not initialized. Use from_dict_with_inference_key or set_inference_client to initialize inference capabilities.'
|
|
445
|
+
)
|
|
446
446
|
|
|
447
|
-
def run_sync(
|
|
447
|
+
def run_sync(
|
|
448
|
+
self,
|
|
449
|
+
data: Dict[str, Any],
|
|
450
|
+
path: str = '',
|
|
451
|
+
timeout_seconds: int = 60 * 5,
|
|
452
|
+
headers: Optional[Dict[str, str]] = None,
|
|
453
|
+
http_method: str = 'POST',
|
|
454
|
+
stream: bool = False,
|
|
455
|
+
) -> InferenceResponse:
|
|
448
456
|
"""Runs a synchronous inference request.
|
|
449
457
|
|
|
450
458
|
Args:
|
|
@@ -462,9 +470,19 @@ class Deployment:
|
|
|
462
470
|
ValueError: If the inference client is not initialized.
|
|
463
471
|
"""
|
|
464
472
|
self._validate_inference_client()
|
|
465
|
-
return self._inference_client.run_sync(
|
|
473
|
+
return self._inference_client.run_sync(
|
|
474
|
+
data, path, timeout_seconds, headers, http_method, stream
|
|
475
|
+
)
|
|
466
476
|
|
|
467
|
-
def run(
|
|
477
|
+
def run(
|
|
478
|
+
self,
|
|
479
|
+
data: Dict[str, Any],
|
|
480
|
+
path: str = '',
|
|
481
|
+
timeout_seconds: int = 60 * 5,
|
|
482
|
+
headers: Optional[Dict[str, str]] = None,
|
|
483
|
+
http_method: str = 'POST',
|
|
484
|
+
stream: bool = False,
|
|
485
|
+
):
|
|
468
486
|
"""Runs an asynchronous inference request.
|
|
469
487
|
|
|
470
488
|
Args:
|
|
@@ -495,11 +513,16 @@ class Deployment:
|
|
|
495
513
|
"""
|
|
496
514
|
self._validate_inference_client()
|
|
497
515
|
# build healthcheck path
|
|
498
|
-
healthcheck_path =
|
|
499
|
-
if
|
|
516
|
+
healthcheck_path = '/health'
|
|
517
|
+
if (
|
|
518
|
+
self.containers
|
|
519
|
+
and self.containers[0].healthcheck
|
|
520
|
+
and self.containers[0].healthcheck.path
|
|
521
|
+
):
|
|
500
522
|
healthcheck_path = self.containers[0].healthcheck.path
|
|
501
523
|
|
|
502
524
|
return self._inference_client.health(healthcheck_path)
|
|
525
|
+
|
|
503
526
|
# Function alias
|
|
504
527
|
healthcheck = health
|
|
505
528
|
|
|
@@ -655,7 +678,14 @@ class AWSECRCredentials(BaseRegistryCredentials):
|
|
|
655
678
|
region: str
|
|
656
679
|
ecr_repo: str
|
|
657
680
|
|
|
658
|
-
def __init__(
|
|
681
|
+
def __init__(
|
|
682
|
+
self,
|
|
683
|
+
name: str,
|
|
684
|
+
access_key_id: str,
|
|
685
|
+
secret_access_key: str,
|
|
686
|
+
region: str,
|
|
687
|
+
ecr_repo: str,
|
|
688
|
+
):
|
|
659
689
|
"""Initializes AWS ECR credentials.
|
|
660
690
|
|
|
661
691
|
Args:
|
|
@@ -718,7 +748,10 @@ class ContainersService:
|
|
|
718
748
|
List[Deployment]: List of all deployments.
|
|
719
749
|
"""
|
|
720
750
|
response = self.client.get(CONTAINER_DEPLOYMENTS_ENDPOINT)
|
|
721
|
-
return [
|
|
751
|
+
return [
|
|
752
|
+
Deployment.from_dict_with_inference_key(deployment, self._inference_key)
|
|
753
|
+
for deployment in response.json()
|
|
754
|
+
]
|
|
722
755
|
|
|
723
756
|
def get_deployment_by_name(self, deployment_name: str) -> Deployment:
|
|
724
757
|
"""Retrieves a specific deployment by name.
|
|
@@ -729,17 +762,13 @@ class ContainersService:
|
|
|
729
762
|
Returns:
|
|
730
763
|
Deployment: The requested deployment.
|
|
731
764
|
"""
|
|
732
|
-
response = self.client.get(
|
|
733
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}")
|
|
765
|
+
response = self.client.get(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}')
|
|
734
766
|
return Deployment.from_dict_with_inference_key(response.json(), self._inference_key)
|
|
735
767
|
|
|
736
768
|
# Function alias
|
|
737
769
|
get_deployment = get_deployment_by_name
|
|
738
770
|
|
|
739
|
-
def create_deployment(
|
|
740
|
-
self,
|
|
741
|
-
deployment: Deployment
|
|
742
|
-
) -> Deployment:
|
|
771
|
+
def create_deployment(self, deployment: Deployment) -> Deployment:
|
|
743
772
|
"""Creates a new container deployment.
|
|
744
773
|
|
|
745
774
|
Args:
|
|
@@ -748,10 +777,7 @@ class ContainersService:
|
|
|
748
777
|
Returns:
|
|
749
778
|
Deployment: The created deployment.
|
|
750
779
|
"""
|
|
751
|
-
response = self.client.post(
|
|
752
|
-
CONTAINER_DEPLOYMENTS_ENDPOINT,
|
|
753
|
-
deployment.to_dict()
|
|
754
|
-
)
|
|
780
|
+
response = self.client.post(CONTAINER_DEPLOYMENTS_ENDPOINT, deployment.to_dict())
|
|
755
781
|
return Deployment.from_dict_with_inference_key(response.json(), self._inference_key)
|
|
756
782
|
|
|
757
783
|
def update_deployment(self, deployment_name: str, deployment: Deployment) -> Deployment:
|
|
@@ -765,8 +791,7 @@ class ContainersService:
|
|
|
765
791
|
Deployment: The updated deployment.
|
|
766
792
|
"""
|
|
767
793
|
response = self.client.patch(
|
|
768
|
-
f
|
|
769
|
-
deployment.to_dict()
|
|
794
|
+
f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}', deployment.to_dict()
|
|
770
795
|
)
|
|
771
796
|
return Deployment.from_dict_with_inference_key(response.json(), self._inference_key)
|
|
772
797
|
|
|
@@ -776,8 +801,7 @@ class ContainersService:
|
|
|
776
801
|
Args:
|
|
777
802
|
deployment_name: Name of the deployment to delete.
|
|
778
803
|
"""
|
|
779
|
-
self.client.delete(
|
|
780
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}")
|
|
804
|
+
self.client.delete(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}')
|
|
781
805
|
|
|
782
806
|
def get_deployment_status(self, deployment_name: str) -> ContainerDeploymentStatus:
|
|
783
807
|
"""Retrieves the current status of a deployment.
|
|
@@ -788,9 +812,8 @@ class ContainersService:
|
|
|
788
812
|
Returns:
|
|
789
813
|
ContainerDeploymentStatus: Current status of the deployment.
|
|
790
814
|
"""
|
|
791
|
-
response = self.client.get(
|
|
792
|
-
|
|
793
|
-
return ContainerDeploymentStatus(response.json()["status"])
|
|
815
|
+
response = self.client.get(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/status')
|
|
816
|
+
return ContainerDeploymentStatus(response.json()['status'])
|
|
794
817
|
|
|
795
818
|
def restart_deployment(self, deployment_name: str) -> None:
|
|
796
819
|
"""Restarts a deployment.
|
|
@@ -798,8 +821,7 @@ class ContainersService:
|
|
|
798
821
|
Args:
|
|
799
822
|
deployment_name: Name of the deployment to restart.
|
|
800
823
|
"""
|
|
801
|
-
self.client.post(
|
|
802
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/restart")
|
|
824
|
+
self.client.post(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/restart')
|
|
803
825
|
|
|
804
826
|
def get_deployment_scaling_options(self, deployment_name: str) -> ScalingOptions:
|
|
805
827
|
"""Retrieves the scaling options for a deployment.
|
|
@@ -810,11 +832,12 @@ class ContainersService:
|
|
|
810
832
|
Returns:
|
|
811
833
|
ScalingOptions: Current scaling options for the deployment.
|
|
812
834
|
"""
|
|
813
|
-
response = self.client.get(
|
|
814
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/scaling")
|
|
835
|
+
response = self.client.get(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/scaling')
|
|
815
836
|
return ScalingOptions.from_dict(response.json())
|
|
816
837
|
|
|
817
|
-
def update_deployment_scaling_options(
|
|
838
|
+
def update_deployment_scaling_options(
|
|
839
|
+
self, deployment_name: str, scaling_options: ScalingOptions
|
|
840
|
+
) -> ScalingOptions:
|
|
818
841
|
"""Updates the scaling options for a deployment.
|
|
819
842
|
|
|
820
843
|
Args:
|
|
@@ -825,8 +848,8 @@ class ContainersService:
|
|
|
825
848
|
ScalingOptions: Updated scaling options for the deployment.
|
|
826
849
|
"""
|
|
827
850
|
response = self.client.patch(
|
|
828
|
-
f
|
|
829
|
-
scaling_options.to_dict()
|
|
851
|
+
f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/scaling',
|
|
852
|
+
scaling_options.to_dict(),
|
|
830
853
|
)
|
|
831
854
|
return ScalingOptions.from_dict(response.json())
|
|
832
855
|
|
|
@@ -839,9 +862,8 @@ class ContainersService:
|
|
|
839
862
|
Returns:
|
|
840
863
|
List[ReplicaInfo]: List of replica information.
|
|
841
864
|
"""
|
|
842
|
-
response = self.client.get(
|
|
843
|
-
|
|
844
|
-
return [ReplicaInfo.from_dict(replica) for replica in response.json()["list"]]
|
|
865
|
+
response = self.client.get(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/replicas')
|
|
866
|
+
return [ReplicaInfo.from_dict(replica) for replica in response.json()['list']]
|
|
845
867
|
|
|
846
868
|
def purge_deployment_queue(self, deployment_name: str) -> None:
|
|
847
869
|
"""Purges the deployment queue.
|
|
@@ -849,8 +871,7 @@ class ContainersService:
|
|
|
849
871
|
Args:
|
|
850
872
|
deployment_name: Name of the deployment.
|
|
851
873
|
"""
|
|
852
|
-
self.client.post(
|
|
853
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/purge-queue")
|
|
874
|
+
self.client.post(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/purge-queue')
|
|
854
875
|
|
|
855
876
|
def pause_deployment(self, deployment_name: str) -> None:
|
|
856
877
|
"""Pauses a deployment.
|
|
@@ -858,8 +879,7 @@ class ContainersService:
|
|
|
858
879
|
Args:
|
|
859
880
|
deployment_name: Name of the deployment to pause.
|
|
860
881
|
"""
|
|
861
|
-
self.client.post(
|
|
862
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/pause")
|
|
882
|
+
self.client.post(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/pause')
|
|
863
883
|
|
|
864
884
|
def resume_deployment(self, deployment_name: str) -> None:
|
|
865
885
|
"""Resumes a paused deployment.
|
|
@@ -867,8 +887,7 @@ class ContainersService:
|
|
|
867
887
|
Args:
|
|
868
888
|
deployment_name: Name of the deployment to resume.
|
|
869
889
|
"""
|
|
870
|
-
self.client.post(
|
|
871
|
-
f"{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/resume")
|
|
890
|
+
self.client.post(f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/resume')
|
|
872
891
|
|
|
873
892
|
def get_deployment_environment_variables(self, deployment_name: str) -> Dict[str, List[EnvVar]]:
|
|
874
893
|
"""Retrieves environment variables for a deployment.
|
|
@@ -880,16 +899,18 @@ class ContainersService:
|
|
|
880
899
|
Dict[str, List[EnvVar]]: Dictionary mapping container names to their environment variables.
|
|
881
900
|
"""
|
|
882
901
|
response = self.client.get(
|
|
883
|
-
f
|
|
902
|
+
f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/environment-variables'
|
|
903
|
+
)
|
|
884
904
|
result = {}
|
|
885
905
|
for item in response.json():
|
|
886
|
-
container_name = item[
|
|
887
|
-
env_vars = item[
|
|
888
|
-
result[container_name] = [EnvVar.from_dict(
|
|
889
|
-
env_var) for env_var in env_vars]
|
|
906
|
+
container_name = item['container_name']
|
|
907
|
+
env_vars = item['env']
|
|
908
|
+
result[container_name] = [EnvVar.from_dict(env_var) for env_var in env_vars]
|
|
890
909
|
return result
|
|
891
910
|
|
|
892
|
-
def add_deployment_environment_variables(
|
|
911
|
+
def add_deployment_environment_variables(
|
|
912
|
+
self, deployment_name: str, container_name: str, env_vars: List[EnvVar]
|
|
913
|
+
) -> Dict[str, List[EnvVar]]:
|
|
893
914
|
"""Adds environment variables to a container in a deployment.
|
|
894
915
|
|
|
895
916
|
Args:
|
|
@@ -901,19 +922,22 @@ class ContainersService:
|
|
|
901
922
|
Dict[str, List[EnvVar]]: Updated environment variables for all containers.
|
|
902
923
|
"""
|
|
903
924
|
response = self.client.post(
|
|
904
|
-
f
|
|
905
|
-
{
|
|
906
|
-
|
|
925
|
+
f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/environment-variables',
|
|
926
|
+
{
|
|
927
|
+
'container_name': container_name,
|
|
928
|
+
'env': [env_var.to_dict() for env_var in env_vars],
|
|
929
|
+
},
|
|
907
930
|
)
|
|
908
931
|
result = {}
|
|
909
932
|
for item in response.json():
|
|
910
|
-
container_name = item[
|
|
911
|
-
env_vars = item[
|
|
912
|
-
result[container_name] = [EnvVar.from_dict(
|
|
913
|
-
env_var) for env_var in env_vars]
|
|
933
|
+
container_name = item['container_name']
|
|
934
|
+
env_vars = item['env']
|
|
935
|
+
result[container_name] = [EnvVar.from_dict(env_var) for env_var in env_vars]
|
|
914
936
|
return result
|
|
915
937
|
|
|
916
|
-
def update_deployment_environment_variables(
|
|
938
|
+
def update_deployment_environment_variables(
|
|
939
|
+
self, deployment_name: str, container_name: str, env_vars: List[EnvVar]
|
|
940
|
+
) -> Dict[str, List[EnvVar]]:
|
|
917
941
|
"""Updates environment variables for a container in a deployment.
|
|
918
942
|
|
|
919
943
|
Args:
|
|
@@ -925,19 +949,22 @@ class ContainersService:
|
|
|
925
949
|
Dict[str, List[EnvVar]]: Updated environment variables for all containers.
|
|
926
950
|
"""
|
|
927
951
|
response = self.client.patch(
|
|
928
|
-
f
|
|
929
|
-
{
|
|
930
|
-
|
|
952
|
+
f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/environment-variables',
|
|
953
|
+
{
|
|
954
|
+
'container_name': container_name,
|
|
955
|
+
'env': [env_var.to_dict() for env_var in env_vars],
|
|
956
|
+
},
|
|
931
957
|
)
|
|
932
958
|
result = {}
|
|
933
959
|
item = response.json()
|
|
934
|
-
container_name = item[
|
|
935
|
-
env_vars = item[
|
|
936
|
-
result[container_name] = [EnvVar.from_dict(
|
|
937
|
-
env_var) for env_var in env_vars]
|
|
960
|
+
container_name = item['container_name']
|
|
961
|
+
env_vars = item['env']
|
|
962
|
+
result[container_name] = [EnvVar.from_dict(env_var) for env_var in env_vars]
|
|
938
963
|
return result
|
|
939
964
|
|
|
940
|
-
def delete_deployment_environment_variables(
|
|
965
|
+
def delete_deployment_environment_variables(
|
|
966
|
+
self, deployment_name: str, container_name: str, env_var_names: List[str]
|
|
967
|
+
) -> Dict[str, List[EnvVar]]:
|
|
941
968
|
"""Deletes environment variables from a container in a deployment.
|
|
942
969
|
|
|
943
970
|
Args:
|
|
@@ -949,18 +976,19 @@ class ContainersService:
|
|
|
949
976
|
Dict[str, List[EnvVar]]: Updated environment variables for all containers.
|
|
950
977
|
"""
|
|
951
978
|
response = self.client.delete(
|
|
952
|
-
f
|
|
953
|
-
{
|
|
979
|
+
f'{CONTAINER_DEPLOYMENTS_ENDPOINT}/{deployment_name}/environment-variables',
|
|
980
|
+
{'container_name': container_name, 'env': env_var_names},
|
|
954
981
|
)
|
|
955
982
|
result = {}
|
|
956
983
|
for item in response.json():
|
|
957
|
-
container_name = item[
|
|
958
|
-
env_vars = item[
|
|
959
|
-
result[container_name] = [EnvVar.from_dict(
|
|
960
|
-
env_var) for env_var in env_vars]
|
|
984
|
+
container_name = item['container_name']
|
|
985
|
+
env_vars = item['env']
|
|
986
|
+
result[container_name] = [EnvVar.from_dict(env_var) for env_var in env_vars]
|
|
961
987
|
return result
|
|
962
988
|
|
|
963
|
-
def get_compute_resources(
|
|
989
|
+
def get_compute_resources(
|
|
990
|
+
self, size: int = None, is_available: bool = None
|
|
991
|
+
) -> List[ComputeResource]:
|
|
964
992
|
"""Retrieves compute resources, optionally filtered by size and availability.
|
|
965
993
|
|
|
966
994
|
Args:
|
|
@@ -979,8 +1007,7 @@ class ContainersService:
|
|
|
979
1007
|
if size:
|
|
980
1008
|
resources = [r for r in resources if r.size == size]
|
|
981
1009
|
if is_available:
|
|
982
|
-
resources = [
|
|
983
|
-
r for r in resources if r.is_available == is_available]
|
|
1010
|
+
resources = [r for r in resources if r.is_available == is_available]
|
|
984
1011
|
return resources
|
|
985
1012
|
|
|
986
1013
|
# Function alias
|
|
@@ -1002,7 +1029,7 @@ class ContainersService:
|
|
|
1002
1029
|
name: Name of the secret.
|
|
1003
1030
|
value: Value of the secret.
|
|
1004
1031
|
"""
|
|
1005
|
-
self.client.post(SECRETS_ENDPOINT, {
|
|
1032
|
+
self.client.post(SECRETS_ENDPOINT, {'name': name, 'value': value})
|
|
1006
1033
|
|
|
1007
1034
|
def delete_secret(self, secret_name: str, force: bool = False) -> None:
|
|
1008
1035
|
"""Deletes a secret.
|
|
@@ -1012,7 +1039,8 @@ class ContainersService:
|
|
|
1012
1039
|
force: Whether to force delete even if secret is in use.
|
|
1013
1040
|
"""
|
|
1014
1041
|
self.client.delete(
|
|
1015
|
-
f
|
|
1042
|
+
f'{SECRETS_ENDPOINT}/{secret_name}', params={'force': str(force).lower()}
|
|
1043
|
+
)
|
|
1016
1044
|
|
|
1017
1045
|
def get_registry_credentials(self) -> List[RegistryCredential]:
|
|
1018
1046
|
"""Retrieves all registry credentials.
|
|
@@ -1038,8 +1066,7 @@ class ContainersService:
|
|
|
1038
1066
|
Args:
|
|
1039
1067
|
credentials_name: Name of the credentials to delete.
|
|
1040
1068
|
"""
|
|
1041
|
-
self.client.delete(
|
|
1042
|
-
f"{CONTAINER_REGISTRY_CREDENTIALS_ENDPOINT}/{credentials_name}")
|
|
1069
|
+
self.client.delete(f'{CONTAINER_REGISTRY_CREDENTIALS_ENDPOINT}/{credentials_name}')
|
|
1043
1070
|
|
|
1044
1071
|
def get_fileset_secrets(self) -> List[Secret]:
|
|
1045
1072
|
"""Retrieves all fileset secrets.
|
|
@@ -1056,11 +1083,13 @@ class ContainersService:
|
|
|
1056
1083
|
Args:
|
|
1057
1084
|
secret_name: Name of the secret to delete.
|
|
1058
1085
|
"""
|
|
1059
|
-
self.client.delete(f
|
|
1086
|
+
self.client.delete(f'{FILESET_SECRETS_ENDPOINT}/{secret_name}')
|
|
1060
1087
|
|
|
1061
|
-
def create_fileset_secret_from_file_paths(
|
|
1088
|
+
def create_fileset_secret_from_file_paths(
|
|
1089
|
+
self, secret_name: str, file_paths: List[str]
|
|
1090
|
+
) -> None:
|
|
1062
1091
|
"""Creates a new fileset secret.
|
|
1063
|
-
A fileset secret is a secret that contains several files,
|
|
1092
|
+
A fileset secret is a secret that contains several files,
|
|
1064
1093
|
and can be used to mount a directory with the files in a container.
|
|
1065
1094
|
|
|
1066
1095
|
Args:
|
|
@@ -1069,13 +1098,12 @@ class ContainersService:
|
|
|
1069
1098
|
"""
|
|
1070
1099
|
processed_files = []
|
|
1071
1100
|
for file_path in file_paths:
|
|
1072
|
-
with open(file_path,
|
|
1073
|
-
base64_content = base64.b64encode(f.read()).decode(
|
|
1074
|
-
processed_files.append(
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
})
|
|
1101
|
+
with open(file_path, 'rb') as f:
|
|
1102
|
+
base64_content = base64.b64encode(f.read()).decode('utf-8')
|
|
1103
|
+
processed_files.append(
|
|
1104
|
+
{
|
|
1105
|
+
'file_name': os.path.basename(file_path),
|
|
1106
|
+
'base64_content': base64_content,
|
|
1107
|
+
}
|
|
1108
|
+
)
|
|
1109
|
+
self.client.post(FILESET_SECRETS_ENDPOINT, {'name': secret_name, 'files': processed_files})
|
datacrunch/datacrunch.py
CHANGED
|
@@ -11,13 +11,19 @@ from datacrunch.volumes.volumes import VolumesService
|
|
|
11
11
|
from datacrunch.containers.containers import ContainersService
|
|
12
12
|
from datacrunch.constants import Constants
|
|
13
13
|
from datacrunch.locations.locations import LocationsService
|
|
14
|
-
from datacrunch.
|
|
14
|
+
from datacrunch._version import __version__
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class DataCrunchClient:
|
|
18
18
|
"""Client for interacting with DataCrunch's public API"""
|
|
19
19
|
|
|
20
|
-
def __init__(
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
client_id: str,
|
|
23
|
+
client_secret: str,
|
|
24
|
+
base_url: str = 'https://api.datacrunch.io/v1',
|
|
25
|
+
inference_key: str = None,
|
|
26
|
+
) -> None:
|
|
21
27
|
"""The DataCrunch client
|
|
22
28
|
|
|
23
29
|
:param client_id: client id
|
|
@@ -32,18 +38,17 @@ class DataCrunchClient:
|
|
|
32
38
|
|
|
33
39
|
# Validate that client_id and client_secret are not empty
|
|
34
40
|
if not client_id or not client_secret:
|
|
35
|
-
raise ValueError(
|
|
36
|
-
"client_id and client_secret must be provided")
|
|
41
|
+
raise ValueError('client_id and client_secret must be provided')
|
|
37
42
|
|
|
38
43
|
# Constants
|
|
39
|
-
self.constants: Constants = Constants(base_url,
|
|
44
|
+
self.constants: Constants = Constants(base_url, __version__)
|
|
40
45
|
"""Constants"""
|
|
41
46
|
|
|
42
47
|
# Services
|
|
43
48
|
self._authentication: AuthenticationService = AuthenticationService(
|
|
44
|
-
client_id, client_secret, self.constants.base_url
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
client_id, client_secret, self.constants.base_url
|
|
50
|
+
)
|
|
51
|
+
self._http_client: HTTPClient = HTTPClient(self._authentication, self.constants.base_url)
|
|
47
52
|
|
|
48
53
|
self.balance: BalanceService = BalanceService(self._http_client)
|
|
49
54
|
"""Balance service. Get client balance"""
|
|
@@ -51,8 +56,7 @@ class DataCrunchClient:
|
|
|
51
56
|
self.images: ImagesService = ImagesService(self._http_client)
|
|
52
57
|
"""Image service"""
|
|
53
58
|
|
|
54
|
-
self.instance_types: InstanceTypesService = InstanceTypesService(
|
|
55
|
-
self._http_client)
|
|
59
|
+
self.instance_types: InstanceTypesService = InstanceTypesService(self._http_client)
|
|
56
60
|
"""Instance type service"""
|
|
57
61
|
|
|
58
62
|
self.instances: InstancesService = InstancesService(self._http_client)
|
|
@@ -61,21 +65,17 @@ class DataCrunchClient:
|
|
|
61
65
|
self.ssh_keys: SSHKeysService = SSHKeysService(self._http_client)
|
|
62
66
|
"""SSH keys service"""
|
|
63
67
|
|
|
64
|
-
self.startup_scripts: StartupScriptsService = StartupScriptsService(
|
|
65
|
-
self._http_client)
|
|
68
|
+
self.startup_scripts: StartupScriptsService = StartupScriptsService(self._http_client)
|
|
66
69
|
"""Startup Scripts service"""
|
|
67
70
|
|
|
68
|
-
self.volume_types: VolumeTypesService = VolumeTypesService(
|
|
69
|
-
self._http_client)
|
|
71
|
+
self.volume_types: VolumeTypesService = VolumeTypesService(self._http_client)
|
|
70
72
|
"""Volume type service"""
|
|
71
73
|
|
|
72
74
|
self.volumes: VolumesService = VolumesService(self._http_client)
|
|
73
75
|
"""Volume service. Create, attach, detach, get, rename, delete volumes"""
|
|
74
76
|
|
|
75
|
-
self.locations: LocationsService = LocationsService(
|
|
76
|
-
self._http_client)
|
|
77
|
+
self.locations: LocationsService = LocationsService(self._http_client)
|
|
77
78
|
"""Locations service. Get locations"""
|
|
78
79
|
|
|
79
|
-
self.containers: ContainersService = ContainersService(
|
|
80
|
-
self._http_client, inference_key)
|
|
80
|
+
self.containers: ContainersService = ContainersService(self._http_client, inference_key)
|
|
81
81
|
"""Containers service. Deploy, manage, and monitor container deployments"""
|
datacrunch/helpers.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from typing import Type
|
|
2
2
|
import json
|
|
3
3
|
|
|
4
|
+
|
|
4
5
|
def stringify_class_object_properties(class_object: Type) -> str:
|
|
5
6
|
"""Generates a json string representation of a class object's properties and values
|
|
6
7
|
|
|
@@ -9,5 +10,9 @@ def stringify_class_object_properties(class_object: Type) -> str:
|
|
|
9
10
|
:return: _description_
|
|
10
11
|
:rtype: json string representation of a class object's properties and values
|
|
11
12
|
"""
|
|
12
|
-
class_properties = {
|
|
13
|
-
|
|
13
|
+
class_properties = {
|
|
14
|
+
property: getattr(class_object, property, '')
|
|
15
|
+
for property in class_object.__dir__()
|
|
16
|
+
if property[:1] != '_' and type(getattr(class_object, property, '')).__name__ != 'method'
|
|
17
|
+
}
|
|
18
|
+
return json.dumps(class_properties, indent=2)
|