dtlpy 1.114.17__py3-none-any.whl → 1.116.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -311
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -296
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -442
  76. dtlpy/entities/dataset.py +1299 -1285
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -223
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +145 -145
  83. dtlpy/entities/filters.py +798 -645
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +959 -953
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -499
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +963 -958
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1257 -1086
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -158
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -435
  166. dtlpy/repositories/datasets.py +1504 -1291
  167. dtlpy/repositories/downloader.py +976 -903
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -470
  170. dtlpy/repositories/executions.py +815 -817
  171. dtlpy/repositories/feature_sets.py +226 -226
  172. dtlpy/repositories/features.py +255 -238
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -909
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -988
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +419 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -651
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1785 -1782
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.114.17.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.114.17.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -183
  230. dtlpy-1.116.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.114.17.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.114.17.dist-info/RECORD +0 -240
  237. {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
dtlpy/entities/compute.py CHANGED
@@ -1,442 +1,449 @@
1
- import traceback
2
- from enum import Enum
3
- from typing import List, Optional, Dict
4
- from ..services.api_client import ApiClient
5
- from .. import repositories
6
-
7
-
8
- class ClusterProvider(str, Enum):
9
- GCP = 'gcp'
10
- AWS = 'aws'
11
- AZURE = 'azure'
12
- HPC = 'hpc'
13
- LOCAL = 'local'
14
- RANCHER_K3S = 'rancher-k3s'
15
- RANCHER_RKE = 'rancher-rke'
16
-
17
-
18
- class ComputeType(str, Enum):
19
- KUBERNETES = "kubernetes"
20
-
21
-
22
- class ComputeStatus(str, Enum):
23
- READY = "ready"
24
- INITIALIZING = "initializing"
25
- PAUSE = "pause"
26
- FAILED = "failed"
27
- VALIDATING = "validating"
28
-
29
-
30
- class ComputeConsumptionMethod(str, Enum):
31
- MQ = "MQ",
32
- API = "API"
33
-
34
-
35
- class ComputeSettings:
36
- def __init__(self, default_namespace: str, consumption_method: ComputeConsumptionMethod):
37
- self.consumption_method = consumption_method
38
- self.default_namespace = default_namespace
39
-
40
- @classmethod
41
- def from_json(cls, _json):
42
- return cls(
43
- default_namespace=_json.get('defaultNamespace'),
44
- consumption_method=_json.get('consumptionMethod')
45
- )
46
-
47
- def to_json(self):
48
- return {
49
- 'defaultNamespace': self.default_namespace,
50
- 'consumptionMethod': self.consumption_method
51
- }
52
-
53
-
54
- class Toleration:
55
- def __init__(self, effect: str, key: str, operator: str, value: str):
56
- self.effect = effect
57
- self.key = key
58
- self.operator = operator
59
- self.value = value
60
-
61
- @classmethod
62
- def from_json(cls, _json):
63
- return cls(
64
- effect=_json.get('effect'),
65
- key=_json.get('key'),
66
- operator=_json.get('operator'),
67
- value=_json.get('value')
68
- )
69
-
70
- def to_json(self):
71
- return {
72
- 'effect': self.effect,
73
- 'key': self.key,
74
- 'operator': self.operator,
75
- 'value': self.value
76
- }
77
-
78
-
79
- class DeploymentResource:
80
- def __init__(self, gpu: int, cpu: int, memory: str):
81
- self.gpu = gpu
82
- self.cpu = cpu
83
- self.memory = memory
84
-
85
- @classmethod
86
- def from_json(cls, _json):
87
- return cls(
88
- gpu=_json.get('gpu', None),
89
- cpu=_json.get('cpu', None),
90
- memory=_json.get('memory', None)
91
- )
92
-
93
- def to_json(self):
94
- _json = {}
95
- if self.gpu is not None:
96
- _json['gpu'] = self.gpu
97
- if self.cpu is not None:
98
- _json['cpu'] = self.cpu
99
- if self.memory is not None:
100
- _json['memory'] = self.memory
101
-
102
-
103
- class DeploymentResources:
104
- def __init__(self, request: DeploymentResource, limit: DeploymentResource):
105
- self.request = request
106
- self.limit = limit
107
-
108
- @classmethod
109
- def from_json(cls, _json):
110
- return cls(
111
- request=DeploymentResource.from_json(_json.get('request') or dict()),
112
- limit=DeploymentResource.from_json(_json.get('limit') or dict())
113
- )
114
-
115
- def to_json(self):
116
- return {
117
- 'request': self.request.to_json(),
118
- 'limit': self.limit.to_json()
119
- }
120
-
121
-
122
- class NodePool:
123
- def __init__(
124
- self,
125
- name: str,
126
- is_dl_type_default: bool,
127
- dl_types: Optional[List[str]] = None,
128
- tolerations: Optional[List[Toleration]] = None,
129
- description: str = "",
130
- node_selector: str = "",
131
- preemptible: bool = False,
132
- deployment_resources: DeploymentResources = None
133
- ):
134
- self.name = name
135
- self.is_dl_type_default = is_dl_type_default
136
- self.dl_types = dl_types
137
- self.tolerations = tolerations if tolerations is not None else []
138
- self.description = description
139
- self.node_selector = node_selector
140
- self.preemptible = preemptible
141
- self.deployment_resources = deployment_resources
142
-
143
- @classmethod
144
- def from_json(cls, _json):
145
- node_pool = cls(
146
- name=_json.get('name'),
147
- is_dl_type_default=_json.get('isDlTypeDefault'),
148
- dl_types=_json.get('dlTypes'),
149
- description=_json.get('description'),
150
- node_selector=_json.get('nodeSelector'),
151
- preemptible=_json.get('preemptible'),
152
- deployment_resources=DeploymentResources.from_json(_json.get('deploymentResources', dict())),
153
- tolerations=[Toleration.from_json(t) for t in _json.get('tolerations', list())]
154
- )
155
-
156
- return node_pool
157
-
158
- def to_json(self):
159
- _json = {
160
- 'name': self.name,
161
- 'isDlTypeDefault': self.is_dl_type_default,
162
- 'description': self.description,
163
- 'nodeSelector': self.node_selector,
164
- 'preemptible': self.preemptible,
165
- 'deploymentResources': self.deployment_resources.to_json(),
166
- 'tolerations': [t.to_json() for t in self.tolerations]
167
- }
168
-
169
- if self.dl_types is not None:
170
- _json['dlTypes'] = self.dl_types
171
-
172
- return _json
173
-
174
-
175
- class AuthenticationIntegration:
176
- def __init__(self, id: str, type: str):
177
- self.id = id
178
- self.type = type
179
-
180
- @classmethod
181
- def from_json(cls, _json):
182
- return cls(
183
- id=_json.get('id'),
184
- type=_json.get('type')
185
- )
186
-
187
- def to_json(self):
188
- return {
189
- 'id': self.id,
190
- 'type': self.type
191
- }
192
-
193
-
194
- class Authentication:
195
- def __init__(self, integration: AuthenticationIntegration):
196
- self.integration = integration
197
-
198
- @classmethod
199
- def from_json(cls, _json):
200
- return cls(
201
- integration=AuthenticationIntegration.from_json(_json.get('integration', dict()))
202
- )
203
-
204
- def to_json(self):
205
- return {
206
- 'integration': self.integration.to_json()
207
- }
208
-
209
-
210
- class ComputeCluster:
211
- def __init__(
212
- self,
213
- name: str,
214
- endpoint: str,
215
- kubernetes_version: str,
216
- provider: ClusterProvider,
217
- node_pools: Optional[List[NodePool]] = None,
218
- metadata: Optional[Dict] = None,
219
- authentication: Optional[Authentication] = None,
220
- plugins: Optional[dict] = None
221
- ):
222
- self.name = name
223
- self.endpoint = endpoint
224
- self.kubernetes_version = kubernetes_version
225
- self.provider = provider
226
- self.node_pools = node_pools if node_pools is not None else []
227
- self.metadata = metadata if metadata is not None else {}
228
- self.authentication = authentication if authentication is not None else Authentication(
229
- AuthenticationIntegration("", ""))
230
- self.plugins = plugins
231
-
232
- @classmethod
233
- def from_json(cls, _json):
234
- return cls(
235
- name=_json.get('name'),
236
- endpoint=_json.get('endpoint'),
237
- kubernetes_version=_json.get('kubernetesVersion'),
238
- provider=ClusterProvider(_json.get('provider')),
239
- node_pools=[NodePool.from_json(np) for np in _json.get('nodePools', list())],
240
- metadata=_json.get('metadata'),
241
- authentication=Authentication.from_json(_json.get('authentication', dict())),
242
- plugins=_json.get('plugins')
243
- )
244
-
245
- def to_json(self):
246
- return {
247
- 'name': self.name,
248
- 'endpoint': self.endpoint,
249
- 'kubernetesVersion': self.kubernetes_version,
250
- 'provider': self.provider.value,
251
- 'nodePools': [np.to_json() for np in self.node_pools],
252
- 'metadata': self.metadata,
253
- 'authentication': self.authentication.to_json(),
254
- 'plugins': self.plugins
255
- }
256
-
257
- @classmethod
258
- def from_setup_json(cls, devops_output, integration):
259
- node_pools = [NodePool.from_json(n) for n in devops_output['config']['nodePools']]
260
- return cls(
261
- name=devops_output['config']['name'],
262
- endpoint=devops_output['config']['endpoint'],
263
- kubernetes_version=devops_output['config']['kubernetesVersion'],
264
- provider=ClusterProvider(devops_output['config']['provider']),
265
- node_pools=node_pools,
266
- metadata={},
267
- authentication=Authentication(AuthenticationIntegration(integration.id, integration.type)),
268
- plugins=devops_output['config'].get('plugins')
269
- )
270
-
271
-
272
- class ComputeContext:
273
- def __init__(self, labels: List[str], org: str, project: Optional[str] = None):
274
- self.labels = labels
275
- self.org = org
276
- self.project = project
277
-
278
- @classmethod
279
- def from_json(cls, _json):
280
- return cls(
281
- labels=_json.get('labels', list()),
282
- org=_json.get('org'),
283
- project=_json.get('project')
284
- )
285
-
286
- def to_json(self):
287
- return {
288
- 'labels': self.labels,
289
- 'org': self.org,
290
- 'project': self.project
291
- }
292
-
293
-
294
- class Compute:
295
- def __init__(
296
- self,
297
- id: str,
298
- name: str,
299
- context: ComputeContext,
300
- client_api: ApiClient,
301
- shared_contexts: Optional[List[ComputeContext]] = None,
302
- global_: Optional[bool] = None,
303
- status: ComputeStatus = ComputeStatus.INITIALIZING,
304
- type: ComputeType = ComputeType.KUBERNETES,
305
- features: Optional[Dict] = None,
306
- metadata: Optional[Dict] = None,
307
- settings: Optional[ComputeSettings] = None,
308
- url: Optional[str] = None
309
- ):
310
- self.id = id
311
- self.name = name
312
- self.context = context
313
- self.shared_contexts = shared_contexts if shared_contexts is not None else []
314
- self.global_ = global_
315
- self.status = status
316
- self.type = type
317
- self.features = features if features is not None else dict()
318
- self.metadata = metadata if metadata is not None else dict()
319
- self._client_api = client_api
320
- self._computes = None
321
- self._serviceDrivers = None
322
- self.settings = settings
323
- self.url = url
324
-
325
- @property
326
- def computes(self):
327
- if self._computes is None:
328
- self._computes = repositories.Computes(client_api=self._client_api)
329
- return self._computes
330
-
331
- def delete(self, skip_destroy: bool = False):
332
- return self.computes.delete(compute_id=self.id, skip_destroy=skip_destroy)
333
-
334
- def update(self):
335
- return self.computes.update(compute=self)
336
-
337
- @staticmethod
338
- def _protected_from_json(_json: dict, client_api: ApiClient):
339
- """
340
- Same as from_json but with try-except to catch if error
341
-
342
- :param _json: platform json
343
- :param client_api: ApiClient entity
344
- :return:
345
- """
346
- try:
347
- compute = Compute.from_json(_json=_json,
348
- client_api=client_api)
349
- status = True
350
- except Exception:
351
- compute = traceback.format_exc()
352
- status = False
353
- return status, compute
354
-
355
- @classmethod
356
- def from_json(cls, _json, client_api: ApiClient):
357
- return cls(
358
- id=_json.get('id'),
359
- name=_json.get('name'),
360
- context=ComputeContext.from_json(_json.get('context', dict())),
361
- shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
362
- global_=_json.get('global'),
363
- status=ComputeStatus(_json.get('status')),
364
- type=ComputeType(_json.get('type')),
365
- features=_json.get('features'),
366
- client_api=client_api,
367
- metadata=_json.get('metadata'),
368
- settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
369
- url=_json.get('url'),
370
- )
371
-
372
- def to_json(self):
373
- return {
374
- 'id': self.id,
375
- 'name': self.name,
376
- 'context': self.context.to_json(),
377
- 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
378
- 'global': self.global_,
379
- 'status': self.status.value,
380
- 'type': self.type.value,
381
- 'features': self.features,
382
- 'metadata': self.metadata,
383
- 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
384
- 'url': self.url
385
- }
386
-
387
-
388
- class KubernetesCompute(Compute):
389
- def __init__(
390
- self,
391
- id: str,
392
- name: str,
393
- context: ComputeContext,
394
- cluster: ComputeCluster,
395
- shared_contexts: Optional[List[ComputeContext]] = None,
396
- global_: Optional[bool] = None,
397
- status: ComputeStatus = ComputeStatus.INITIALIZING,
398
- type: ComputeType = ComputeType.KUBERNETES,
399
- features: Optional[Dict] = None,
400
- metadata: Optional[Dict] = None,
401
- client_api: ApiClient = None,
402
- settings: Optional[ComputeSettings] = None,
403
- url: Optional[str] = None
404
- ):
405
- super().__init__(id=id, context=context, shared_contexts=shared_contexts, global_=global_, status=status,
406
- type=type, features=features, metadata=metadata, client_api=client_api, settings=settings,
407
- name=name, url=url)
408
- self.cluster = cluster
409
-
410
- @classmethod
411
- def from_json(cls, _json, client_api: ApiClient):
412
- return cls(
413
- id=_json.get('id'),
414
- name=_json.get('name'),
415
- context=ComputeContext.from_json(_json.get('context', dict())),
416
- cluster=ComputeCluster.from_json(_json.get('cluster', dict())),
417
- shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
418
- global_=_json.get('global'),
419
- status=ComputeStatus(_json.get('status')),
420
- type=ComputeType(_json.get('type')),
421
- features=_json.get('features'),
422
- metadata=_json.get('metadata'),
423
- client_api=client_api,
424
- settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
425
- url=_json.get('url'),
426
- )
427
-
428
- def to_json(self):
429
- return {
430
- 'id': self.id,
431
- 'name': self.name,
432
- 'context': self.context.to_json(),
433
- 'cluster': self.cluster.to_json(),
434
- 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
435
- 'global': self.global_,
436
- 'status': self.status.value,
437
- 'type': self.type.value,
438
- 'features': self.features,
439
- 'metadata': self.metadata,
440
- 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
441
- 'url': self.url
442
- }
1
+ import traceback
2
+ from enum import Enum
3
+ from typing import List, Optional, Dict
4
+ from ..services.api_client import ApiClient
5
+ from .. import repositories
6
+
7
+
8
+ class ClusterProvider(str, Enum):
9
+ GCP = 'gcp'
10
+ AWS = 'aws'
11
+ AZURE = 'azure'
12
+ HPC = 'hpc'
13
+ LOCAL = 'local'
14
+ RANCHER_K3S = 'rancher-k3s'
15
+ RANCHER_RKE = 'rancher-rke'
16
+
17
+
18
+ class ComputeType(str, Enum):
19
+ KUBERNETES = "kubernetes"
20
+
21
+
22
+ class ComputeStatus(str, Enum):
23
+ READY = "ready"
24
+ INITIALIZING = "initializing"
25
+ PAUSE = "pause"
26
+ FAILED = "failed"
27
+ VALIDATING = "validating"
28
+
29
+
30
+ class ComputeConsumptionMethod(str, Enum):
31
+ MQ = "MQ",
32
+ API = "API"
33
+
34
+
35
+ class ComputeSettings:
36
+ def __init__(self, default_namespace: str, consumption_method: ComputeConsumptionMethod):
37
+ self.consumption_method = consumption_method
38
+ self.default_namespace = default_namespace
39
+
40
+ @classmethod
41
+ def from_json(cls, _json):
42
+ return cls(
43
+ default_namespace=_json.get('defaultNamespace'),
44
+ consumption_method=_json.get('consumptionMethod')
45
+ )
46
+
47
+ def to_json(self):
48
+ return {
49
+ 'defaultNamespace': self.default_namespace,
50
+ 'consumptionMethod': self.consumption_method
51
+ }
52
+
53
+
54
+ class Toleration:
55
+ def __init__(self, effect: str, key: str, operator: str, value: str):
56
+ self.effect = effect
57
+ self.key = key
58
+ self.operator = operator
59
+ self.value = value
60
+
61
+ @classmethod
62
+ def from_json(cls, _json):
63
+ return cls(
64
+ effect=_json.get('effect'),
65
+ key=_json.get('key'),
66
+ operator=_json.get('operator'),
67
+ value=_json.get('value')
68
+ )
69
+
70
+ def to_json(self):
71
+ return {
72
+ 'effect': self.effect,
73
+ 'key': self.key,
74
+ 'operator': self.operator,
75
+ 'value': self.value
76
+ }
77
+
78
+
79
+ class DeploymentResource:
80
+ def __init__(self, gpu: int, cpu: int, memory: str):
81
+ self.gpu = gpu
82
+ self.cpu = cpu
83
+ self.memory = memory
84
+
85
+ @classmethod
86
+ def from_json(cls, _json):
87
+ return cls(
88
+ gpu=_json.get('gpu', None),
89
+ cpu=_json.get('cpu', None),
90
+ memory=_json.get('memory', None)
91
+ )
92
+
93
+ def to_json(self):
94
+ _json = {}
95
+ if self.gpu is not None:
96
+ _json['gpu'] = self.gpu
97
+ if self.cpu is not None:
98
+ _json['cpu'] = self.cpu
99
+ if self.memory is not None:
100
+ _json['memory'] = self.memory
101
+
102
+
103
+ class DeploymentResources:
104
+ def __init__(self, request: DeploymentResource, limit: DeploymentResource):
105
+ self.request = request
106
+ self.limit = limit
107
+
108
+ @classmethod
109
+ def from_json(cls, _json):
110
+ return cls(
111
+ request=DeploymentResource.from_json(_json.get('request') or dict()),
112
+ limit=DeploymentResource.from_json(_json.get('limit') or dict())
113
+ )
114
+
115
+ def to_json(self):
116
+ return {
117
+ 'request': self.request.to_json(),
118
+ 'limit': self.limit.to_json()
119
+ }
120
+
121
+
122
+ class NodePool:
123
+ def __init__(
124
+ self,
125
+ name: str,
126
+ is_dl_type_default: bool,
127
+ dl_types: Optional[List[str]] = None,
128
+ tolerations: Optional[List[Toleration]] = None,
129
+ description: str = "",
130
+ node_selector: str = "",
131
+ preemptible: bool = False,
132
+ deployment_resources: DeploymentResources = None
133
+ ):
134
+ self.name = name
135
+ self.is_dl_type_default = is_dl_type_default
136
+ self.dl_types = dl_types
137
+ self.tolerations = tolerations if tolerations is not None else []
138
+ self.description = description
139
+ self.node_selector = node_selector
140
+ self.preemptible = preemptible
141
+ self.deployment_resources = deployment_resources
142
+
143
+ @classmethod
144
+ def from_json(cls, _json):
145
+ node_pool = cls(
146
+ name=_json.get('name'),
147
+ is_dl_type_default=_json.get('isDlTypeDefault'),
148
+ dl_types=_json.get('dlTypes'),
149
+ description=_json.get('description'),
150
+ node_selector=_json.get('nodeSelector'),
151
+ preemptible=_json.get('preemptible'),
152
+ deployment_resources=DeploymentResources.from_json(_json.get('deploymentResources', dict())),
153
+ tolerations=[Toleration.from_json(t) for t in _json.get('tolerations', list())]
154
+ )
155
+
156
+ return node_pool
157
+
158
+ def to_json(self):
159
+ _json = {
160
+ 'name': self.name,
161
+ 'isDlTypeDefault': self.is_dl_type_default,
162
+ 'description': self.description,
163
+ 'nodeSelector': self.node_selector,
164
+ 'preemptible': self.preemptible,
165
+ 'deploymentResources': self.deployment_resources.to_json(),
166
+ 'tolerations': [t.to_json() for t in self.tolerations]
167
+ }
168
+
169
+ if self.dl_types is not None:
170
+ _json['dlTypes'] = self.dl_types
171
+
172
+ return _json
173
+
174
+
175
+ class AuthenticationIntegration:
176
+ def __init__(self, id: str, type: str):
177
+ self.id = id
178
+ self.type = type
179
+
180
+ @classmethod
181
+ def from_json(cls, _json):
182
+ return cls(
183
+ id=_json.get('id'),
184
+ type=_json.get('type')
185
+ )
186
+
187
+ def to_json(self):
188
+ return {
189
+ 'id': self.id,
190
+ 'type': self.type
191
+ }
192
+
193
+
194
+ class Authentication:
195
+ def __init__(self, integration: AuthenticationIntegration):
196
+ self.integration = integration
197
+
198
+ @classmethod
199
+ def from_json(cls, _json):
200
+ return cls(
201
+ integration=AuthenticationIntegration.from_json(_json.get('integration', dict()))
202
+ )
203
+
204
+ def to_json(self):
205
+ return {
206
+ 'integration': self.integration.to_json()
207
+ }
208
+
209
+
210
+ class ComputeCluster:
211
+ def __init__(
212
+ self,
213
+ name: str,
214
+ endpoint: str,
215
+ kubernetes_version: str,
216
+ provider: ClusterProvider,
217
+ node_pools: Optional[List[NodePool]] = None,
218
+ metadata: Optional[Dict] = None,
219
+ authentication: Optional[Authentication] = None,
220
+ plugins: Optional[dict] = None,
221
+ deployment_configuration: Optional[Dict] = None
222
+ ):
223
+ self.name = name
224
+ self.endpoint = endpoint
225
+ self.kubernetes_version = kubernetes_version
226
+ self.provider = provider
227
+ self.node_pools = node_pools if node_pools is not None else []
228
+ self.metadata = metadata if metadata is not None else {}
229
+ self.authentication = authentication if authentication is not None else Authentication(
230
+ AuthenticationIntegration("", ""))
231
+ self.plugins = plugins
232
+ self.deployment_configuration = deployment_configuration if deployment_configuration is not None else {}
233
+
234
+
235
+
236
+ @classmethod
237
+ def from_json(cls, _json):
238
+ return cls(
239
+ name=_json.get('name'),
240
+ endpoint=_json.get('endpoint'),
241
+ kubernetes_version=_json.get('kubernetesVersion'),
242
+ provider=ClusterProvider(_json.get('provider')),
243
+ node_pools=[NodePool.from_json(np) for np in _json.get('nodePools', list())],
244
+ metadata=_json.get('metadata'),
245
+ authentication=Authentication.from_json(_json.get('authentication', dict())),
246
+ plugins=_json.get('plugins'),
247
+ deployment_configuration=_json.get('deploymentConfiguration'),
248
+ )
249
+
250
+ def to_json(self):
251
+ return {
252
+ 'name': self.name,
253
+ 'endpoint': self.endpoint,
254
+ 'kubernetesVersion': self.kubernetes_version,
255
+ 'provider': self.provider.value,
256
+ 'nodePools': [np.to_json() for np in self.node_pools],
257
+ 'metadata': self.metadata,
258
+ 'authentication': self.authentication.to_json(),
259
+ 'plugins': self.plugins,
260
+ 'deploymentConfiguration': self.deployment_configuration
261
+ }
262
+
263
+ @classmethod
264
+ def from_setup_json(cls, devops_output, integration):
265
+ node_pools = [NodePool.from_json(n) for n in devops_output['config']['nodePools']]
266
+ return cls(
267
+ name=devops_output['config']['name'],
268
+ endpoint=devops_output['config']['endpoint'],
269
+ kubernetes_version=devops_output['config']['kubernetesVersion'],
270
+ provider=ClusterProvider(devops_output['config']['provider']),
271
+ node_pools=node_pools,
272
+ metadata={},
273
+ authentication=Authentication(AuthenticationIntegration(integration.id, integration.type)),
274
+ plugins=devops_output['config'].get('plugins'),
275
+ deployment_configuration=devops_output['config'].get('deploymentConfiguration', {})
276
+ )
277
+
278
+
279
+ class ComputeContext:
280
+ def __init__(self, labels: List[str], org: str, project: Optional[str] = None):
281
+ self.labels = labels
282
+ self.org = org
283
+ self.project = project
284
+
285
+ @classmethod
286
+ def from_json(cls, _json):
287
+ return cls(
288
+ labels=_json.get('labels', list()),
289
+ org=_json.get('org'),
290
+ project=_json.get('project')
291
+ )
292
+
293
+ def to_json(self):
294
+ return {
295
+ 'labels': self.labels,
296
+ 'org': self.org,
297
+ 'project': self.project
298
+ }
299
+
300
+
301
+ class Compute:
302
+ def __init__(
303
+ self,
304
+ id: str,
305
+ name: str,
306
+ context: ComputeContext,
307
+ client_api: ApiClient,
308
+ shared_contexts: Optional[List[ComputeContext]] = None,
309
+ global_: Optional[bool] = None,
310
+ status: ComputeStatus = ComputeStatus.INITIALIZING,
311
+ type: ComputeType = ComputeType.KUBERNETES,
312
+ features: Optional[Dict] = None,
313
+ metadata: Optional[Dict] = None,
314
+ settings: Optional[ComputeSettings] = None,
315
+ url: Optional[str] = None
316
+ ):
317
+ self.id = id
318
+ self.name = name
319
+ self.context = context
320
+ self.shared_contexts = shared_contexts if shared_contexts is not None else []
321
+ self.global_ = global_
322
+ self.status = status
323
+ self.type = type
324
+ self.features = features if features is not None else dict()
325
+ self.metadata = metadata if metadata is not None else dict()
326
+ self._client_api = client_api
327
+ self._computes = None
328
+ self._serviceDrivers = None
329
+ self.settings = settings
330
+ self.url = url
331
+
332
+ @property
333
+ def computes(self):
334
+ if self._computes is None:
335
+ self._computes = repositories.Computes(client_api=self._client_api)
336
+ return self._computes
337
+
338
+ def delete(self, skip_destroy: bool = False):
339
+ return self.computes.delete(compute_id=self.id, skip_destroy=skip_destroy)
340
+
341
+ def update(self):
342
+ return self.computes.update(compute=self)
343
+
344
+ @staticmethod
345
+ def _protected_from_json(_json: dict, client_api: ApiClient):
346
+ """
347
+ Same as from_json but with try-except to catch if error
348
+
349
+ :param _json: platform json
350
+ :param client_api: ApiClient entity
351
+ :return:
352
+ """
353
+ try:
354
+ compute = Compute.from_json(_json=_json,
355
+ client_api=client_api)
356
+ status = True
357
+ except Exception:
358
+ compute = traceback.format_exc()
359
+ status = False
360
+ return status, compute
361
+
362
+ @classmethod
363
+ def from_json(cls, _json, client_api: ApiClient):
364
+ return cls(
365
+ id=_json.get('id'),
366
+ name=_json.get('name'),
367
+ context=ComputeContext.from_json(_json.get('context', dict())),
368
+ shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
369
+ global_=_json.get('global'),
370
+ status=ComputeStatus(_json.get('status')),
371
+ type=ComputeType(_json.get('type')),
372
+ features=_json.get('features'),
373
+ client_api=client_api,
374
+ metadata=_json.get('metadata'),
375
+ settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
376
+ url=_json.get('url'),
377
+ )
378
+
379
+ def to_json(self):
380
+ return {
381
+ 'id': self.id,
382
+ 'name': self.name,
383
+ 'context': self.context.to_json(),
384
+ 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
385
+ 'global': self.global_,
386
+ 'status': self.status.value,
387
+ 'type': self.type.value,
388
+ 'features': self.features,
389
+ 'metadata': self.metadata,
390
+ 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
391
+ 'url': self.url
392
+ }
393
+
394
+
395
+ class KubernetesCompute(Compute):
396
+ def __init__(
397
+ self,
398
+ id: str,
399
+ name: str,
400
+ context: ComputeContext,
401
+ cluster: ComputeCluster,
402
+ shared_contexts: Optional[List[ComputeContext]] = None,
403
+ global_: Optional[bool] = None,
404
+ status: ComputeStatus = ComputeStatus.INITIALIZING,
405
+ type: ComputeType = ComputeType.KUBERNETES,
406
+ features: Optional[Dict] = None,
407
+ metadata: Optional[Dict] = None,
408
+ client_api: ApiClient = None,
409
+ settings: Optional[ComputeSettings] = None,
410
+ url: Optional[str] = None
411
+ ):
412
+ super().__init__(id=id, context=context, shared_contexts=shared_contexts, global_=global_, status=status,
413
+ type=type, features=features, metadata=metadata, client_api=client_api, settings=settings,
414
+ name=name, url=url)
415
+ self.cluster = cluster
416
+
417
+ @classmethod
418
+ def from_json(cls, _json, client_api: ApiClient):
419
+ return cls(
420
+ id=_json.get('id'),
421
+ name=_json.get('name'),
422
+ context=ComputeContext.from_json(_json.get('context', dict())),
423
+ cluster=ComputeCluster.from_json(_json.get('cluster', dict())),
424
+ shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
425
+ global_=_json.get('global'),
426
+ status=ComputeStatus(_json.get('status')),
427
+ type=ComputeType(_json.get('type')),
428
+ features=_json.get('features'),
429
+ metadata=_json.get('metadata'),
430
+ client_api=client_api,
431
+ settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
432
+ url=_json.get('url')
433
+ )
434
+
435
+ def to_json(self):
436
+ return {
437
+ 'id': self.id,
438
+ 'name': self.name,
439
+ 'context': self.context.to_json(),
440
+ 'cluster': self.cluster.to_json(),
441
+ 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
442
+ 'global': self.global_,
443
+ 'status': self.status.value,
444
+ 'type': self.type.value,
445
+ 'features': self.features,
446
+ 'metadata': self.metadata,
447
+ 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
448
+ 'url': self.url
449
+ }