dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
dtlpy/entities/compute.py CHANGED
@@ -1,442 +1,442 @@
1
- import traceback
2
- from enum import Enum
3
- from typing import List, Optional, Dict
4
- from ..services.api_client import ApiClient
5
- from .. import repositories
6
-
7
-
8
- class ClusterProvider(str, Enum):
9
- GCP = 'gcp'
10
- AWS = 'aws'
11
- AZURE = 'azure'
12
- HPC = 'hpc'
13
- LOCAL = 'local'
14
- RANCHER_K3S = 'rancher-k3s'
15
- RANCHER_RKE = 'rancher-rke'
16
-
17
-
18
- class ComputeType(str, Enum):
19
- KUBERNETES = "kubernetes"
20
-
21
-
22
- class ComputeStatus(str, Enum):
23
- READY = "ready"
24
- INITIALIZING = "initializing"
25
- PAUSE = "pause"
26
- FAILED = "failed"
27
- VALIDATING = "validating"
28
-
29
-
30
- class ComputeConsumptionMethod(str, Enum):
31
- MQ = "MQ",
32
- API = "API"
33
-
34
-
35
- class ComputeSettings:
36
- def __init__(self, default_namespace: str, consumption_method: ComputeConsumptionMethod):
37
- self.consumption_method = consumption_method
38
- self.default_namespace = default_namespace
39
-
40
- @classmethod
41
- def from_json(cls, _json):
42
- return cls(
43
- default_namespace=_json.get('defaultNamespace'),
44
- consumption_method=_json.get('consumptionMethod')
45
- )
46
-
47
- def to_json(self):
48
- return {
49
- 'defaultNamespace': self.default_namespace,
50
- 'consumptionMethod': self.consumption_method
51
- }
52
-
53
-
54
- class Toleration:
55
- def __init__(self, effect: str, key: str, operator: str, value: str):
56
- self.effect = effect
57
- self.key = key
58
- self.operator = operator
59
- self.value = value
60
-
61
- @classmethod
62
- def from_json(cls, _json):
63
- return cls(
64
- effect=_json.get('effect'),
65
- key=_json.get('key'),
66
- operator=_json.get('operator'),
67
- value=_json.get('value')
68
- )
69
-
70
- def to_json(self):
71
- return {
72
- 'effect': self.effect,
73
- 'key': self.key,
74
- 'operator': self.operator,
75
- 'value': self.value
76
- }
77
-
78
-
79
- class DeploymentResource:
80
- def __init__(self, gpu: int, cpu: int, memory: str):
81
- self.gpu = gpu
82
- self.cpu = cpu
83
- self.memory = memory
84
-
85
- @classmethod
86
- def from_json(cls, _json):
87
- return cls(
88
- gpu=_json.get('gpu', None),
89
- cpu=_json.get('cpu', None),
90
- memory=_json.get('memory', None)
91
- )
92
-
93
- def to_json(self):
94
- _json = {}
95
- if self.gpu is not None:
96
- _json['gpu'] = self.gpu
97
- if self.cpu is not None:
98
- _json['cpu'] = self.cpu
99
- if self.memory is not None:
100
- _json['memory'] = self.memory
101
-
102
-
103
- class DeploymentResources:
104
- def __init__(self, request: DeploymentResource, limit: DeploymentResource):
105
- self.request = request
106
- self.limit = limit
107
-
108
- @classmethod
109
- def from_json(cls, _json):
110
- return cls(
111
- request=DeploymentResource.from_json(_json.get('request') or dict()),
112
- limit=DeploymentResource.from_json(_json.get('limit') or dict())
113
- )
114
-
115
- def to_json(self):
116
- return {
117
- 'request': self.request.to_json(),
118
- 'limit': self.limit.to_json()
119
- }
120
-
121
-
122
- class NodePool:
123
- def __init__(
124
- self,
125
- name: str,
126
- is_dl_type_default: bool,
127
- dl_types: Optional[List[str]] = None,
128
- tolerations: Optional[List[Toleration]] = None,
129
- description: str = "",
130
- node_selector: str = "",
131
- preemptible: bool = False,
132
- deployment_resources: DeploymentResources = None
133
- ):
134
- self.name = name
135
- self.is_dl_type_default = is_dl_type_default
136
- self.dl_types = dl_types
137
- self.tolerations = tolerations if tolerations is not None else []
138
- self.description = description
139
- self.node_selector = node_selector
140
- self.preemptible = preemptible
141
- self.deployment_resources = deployment_resources
142
-
143
- @classmethod
144
- def from_json(cls, _json):
145
- node_pool = cls(
146
- name=_json.get('name'),
147
- is_dl_type_default=_json.get('isDlTypeDefault'),
148
- dl_types=_json.get('dlTypes'),
149
- description=_json.get('description'),
150
- node_selector=_json.get('nodeSelector'),
151
- preemptible=_json.get('preemptible'),
152
- deployment_resources=DeploymentResources.from_json(_json.get('deploymentResources', dict())),
153
- tolerations=[Toleration.from_json(t) for t in _json.get('tolerations', list())]
154
- )
155
-
156
- return node_pool
157
-
158
- def to_json(self):
159
- _json = {
160
- 'name': self.name,
161
- 'isDlTypeDefault': self.is_dl_type_default,
162
- 'description': self.description,
163
- 'nodeSelector': self.node_selector,
164
- 'preemptible': self.preemptible,
165
- 'deploymentResources': self.deployment_resources.to_json(),
166
- 'tolerations': [t.to_json() for t in self.tolerations]
167
- }
168
-
169
- if self.dl_types is not None:
170
- _json['dlTypes'] = self.dl_types
171
-
172
- return _json
173
-
174
-
175
- class AuthenticationIntegration:
176
- def __init__(self, id: str, type: str):
177
- self.id = id
178
- self.type = type
179
-
180
- @classmethod
181
- def from_json(cls, _json):
182
- return cls(
183
- id=_json.get('id'),
184
- type=_json.get('type')
185
- )
186
-
187
- def to_json(self):
188
- return {
189
- 'id': self.id,
190
- 'type': self.type
191
- }
192
-
193
-
194
- class Authentication:
195
- def __init__(self, integration: AuthenticationIntegration):
196
- self.integration = integration
197
-
198
- @classmethod
199
- def from_json(cls, _json):
200
- return cls(
201
- integration=AuthenticationIntegration.from_json(_json.get('integration', dict()))
202
- )
203
-
204
- def to_json(self):
205
- return {
206
- 'integration': self.integration.to_json()
207
- }
208
-
209
-
210
- class ComputeCluster:
211
- def __init__(
212
- self,
213
- name: str,
214
- endpoint: str,
215
- kubernetes_version: str,
216
- provider: ClusterProvider,
217
- node_pools: Optional[List[NodePool]] = None,
218
- metadata: Optional[Dict] = None,
219
- authentication: Optional[Authentication] = None,
220
- plugins: Optional[dict] = None
221
- ):
222
- self.name = name
223
- self.endpoint = endpoint
224
- self.kubernetes_version = kubernetes_version
225
- self.provider = provider
226
- self.node_pools = node_pools if node_pools is not None else []
227
- self.metadata = metadata if metadata is not None else {}
228
- self.authentication = authentication if authentication is not None else Authentication(
229
- AuthenticationIntegration("", ""))
230
- self.plugins = plugins
231
-
232
- @classmethod
233
- def from_json(cls, _json):
234
- return cls(
235
- name=_json.get('name'),
236
- endpoint=_json.get('endpoint'),
237
- kubernetes_version=_json.get('kubernetesVersion'),
238
- provider=ClusterProvider(_json.get('provider')),
239
- node_pools=[NodePool.from_json(np) for np in _json.get('nodePools', list())],
240
- metadata=_json.get('metadata'),
241
- authentication=Authentication.from_json(_json.get('authentication', dict())),
242
- plugins=_json.get('plugins')
243
- )
244
-
245
- def to_json(self):
246
- return {
247
- 'name': self.name,
248
- 'endpoint': self.endpoint,
249
- 'kubernetesVersion': self.kubernetes_version,
250
- 'provider': self.provider.value,
251
- 'nodePools': [np.to_json() for np in self.node_pools],
252
- 'metadata': self.metadata,
253
- 'authentication': self.authentication.to_json(),
254
- 'plugins': self.plugins
255
- }
256
-
257
- @classmethod
258
- def from_setup_json(cls, devops_output, integration):
259
- node_pools = [NodePool.from_json(n) for n in devops_output['config']['nodePools']]
260
- return cls(
261
- name=devops_output['config']['name'],
262
- endpoint=devops_output['config']['endpoint'],
263
- kubernetes_version=devops_output['config']['kubernetesVersion'],
264
- provider=ClusterProvider(devops_output['config']['provider']),
265
- node_pools=node_pools,
266
- metadata={},
267
- authentication=Authentication(AuthenticationIntegration(integration.id, integration.type)),
268
- plugins=devops_output['config'].get('plugins')
269
- )
270
-
271
-
272
- class ComputeContext:
273
- def __init__(self, labels: List[str], org: str, project: Optional[str] = None):
274
- self.labels = labels
275
- self.org = org
276
- self.project = project
277
-
278
- @classmethod
279
- def from_json(cls, _json):
280
- return cls(
281
- labels=_json.get('labels', list()),
282
- org=_json.get('org'),
283
- project=_json.get('project')
284
- )
285
-
286
- def to_json(self):
287
- return {
288
- 'labels': self.labels,
289
- 'org': self.org,
290
- 'project': self.project
291
- }
292
-
293
-
294
- class Compute:
295
- def __init__(
296
- self,
297
- id: str,
298
- name: str,
299
- context: ComputeContext,
300
- client_api: ApiClient,
301
- shared_contexts: Optional[List[ComputeContext]] = None,
302
- global_: Optional[bool] = None,
303
- status: ComputeStatus = ComputeStatus.INITIALIZING,
304
- type: ComputeType = ComputeType.KUBERNETES,
305
- features: Optional[Dict] = None,
306
- metadata: Optional[Dict] = None,
307
- settings: Optional[ComputeSettings] = None,
308
- url: Optional[str] = None
309
- ):
310
- self.id = id
311
- self.name = name
312
- self.context = context
313
- self.shared_contexts = shared_contexts if shared_contexts is not None else []
314
- self.global_ = global_
315
- self.status = status
316
- self.type = type
317
- self.features = features if features is not None else dict()
318
- self.metadata = metadata if metadata is not None else dict()
319
- self._client_api = client_api
320
- self._computes = None
321
- self._serviceDrivers = None
322
- self.settings = settings
323
- self.url = url
324
-
325
- @property
326
- def computes(self):
327
- if self._computes is None:
328
- self._computes = repositories.Computes(client_api=self._client_api)
329
- return self._computes
330
-
331
- def delete(self, skip_destroy: bool = False):
332
- return self.computes.delete(compute_id=self.id, skip_destroy=skip_destroy)
333
-
334
- def update(self):
335
- return self.computes.update(compute=self)
336
-
337
- @staticmethod
338
- def _protected_from_json(_json: dict, client_api: ApiClient):
339
- """
340
- Same as from_json but with try-except to catch if error
341
-
342
- :param _json: platform json
343
- :param client_api: ApiClient entity
344
- :return:
345
- """
346
- try:
347
- compute = Compute.from_json(_json=_json,
348
- client_api=client_api)
349
- status = True
350
- except Exception:
351
- compute = traceback.format_exc()
352
- status = False
353
- return status, compute
354
-
355
- @classmethod
356
- def from_json(cls, _json, client_api: ApiClient):
357
- return cls(
358
- id=_json.get('id'),
359
- name=_json.get('name'),
360
- context=ComputeContext.from_json(_json.get('context', dict())),
361
- shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
362
- global_=_json.get('global'),
363
- status=ComputeStatus(_json.get('status')),
364
- type=ComputeType(_json.get('type')),
365
- features=_json.get('features'),
366
- client_api=client_api,
367
- metadata=_json.get('metadata'),
368
- settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
369
- url=_json.get('url'),
370
- )
371
-
372
- def to_json(self):
373
- return {
374
- 'id': self.id,
375
- 'name': self.name,
376
- 'context': self.context.to_json(),
377
- 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
378
- 'global': self.global_,
379
- 'status': self.status.value,
380
- 'type': self.type.value,
381
- 'features': self.features,
382
- 'metadata': self.metadata,
383
- 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
384
- 'url': self.url
385
- }
386
-
387
-
388
- class KubernetesCompute(Compute):
389
- def __init__(
390
- self,
391
- id: str,
392
- name: str,
393
- context: ComputeContext,
394
- cluster: ComputeCluster,
395
- shared_contexts: Optional[List[ComputeContext]] = None,
396
- global_: Optional[bool] = None,
397
- status: ComputeStatus = ComputeStatus.INITIALIZING,
398
- type: ComputeType = ComputeType.KUBERNETES,
399
- features: Optional[Dict] = None,
400
- metadata: Optional[Dict] = None,
401
- client_api: ApiClient = None,
402
- settings: Optional[ComputeSettings] = None,
403
- url: Optional[str] = None
404
- ):
405
- super().__init__(id=id, context=context, shared_contexts=shared_contexts, global_=global_, status=status,
406
- type=type, features=features, metadata=metadata, client_api=client_api, settings=settings,
407
- name=name, url=url)
408
- self.cluster = cluster
409
-
410
- @classmethod
411
- def from_json(cls, _json, client_api: ApiClient):
412
- return cls(
413
- id=_json.get('id'),
414
- name=_json.get('name'),
415
- context=ComputeContext.from_json(_json.get('context', dict())),
416
- cluster=ComputeCluster.from_json(_json.get('cluster', dict())),
417
- shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
418
- global_=_json.get('global'),
419
- status=ComputeStatus(_json.get('status')),
420
- type=ComputeType(_json.get('type')),
421
- features=_json.get('features'),
422
- metadata=_json.get('metadata'),
423
- client_api=client_api,
424
- settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
425
- url=_json.get('url'),
426
- )
427
-
428
- def to_json(self):
429
- return {
430
- 'id': self.id,
431
- 'name': self.name,
432
- 'context': self.context.to_json(),
433
- 'cluster': self.cluster.to_json(),
434
- 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
435
- 'global': self.global_,
436
- 'status': self.status.value,
437
- 'type': self.type.value,
438
- 'features': self.features,
439
- 'metadata': self.metadata,
440
- 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
441
- 'url': self.url
442
- }
1
+ import traceback
2
+ from enum import Enum
3
+ from typing import List, Optional, Dict
4
+ from ..services.api_client import ApiClient
5
+ from .. import repositories
6
+
7
+
8
+ class ClusterProvider(str, Enum):
9
+ GCP = 'gcp'
10
+ AWS = 'aws'
11
+ AZURE = 'azure'
12
+ HPC = 'hpc'
13
+ LOCAL = 'local'
14
+ RANCHER_K3S = 'rancher-k3s'
15
+ RANCHER_RKE = 'rancher-rke'
16
+
17
+
18
+ class ComputeType(str, Enum):
19
+ KUBERNETES = "kubernetes"
20
+
21
+
22
+ class ComputeStatus(str, Enum):
23
+ READY = "ready"
24
+ INITIALIZING = "initializing"
25
+ PAUSE = "pause"
26
+ FAILED = "failed"
27
+ VALIDATING = "validating"
28
+
29
+
30
+ class ComputeConsumptionMethod(str, Enum):
31
+ MQ = "MQ",
32
+ API = "API"
33
+
34
+
35
+ class ComputeSettings:
36
+ def __init__(self, default_namespace: str, consumption_method: ComputeConsumptionMethod):
37
+ self.consumption_method = consumption_method
38
+ self.default_namespace = default_namespace
39
+
40
+ @classmethod
41
+ def from_json(cls, _json):
42
+ return cls(
43
+ default_namespace=_json.get('defaultNamespace'),
44
+ consumption_method=_json.get('consumptionMethod')
45
+ )
46
+
47
+ def to_json(self):
48
+ return {
49
+ 'defaultNamespace': self.default_namespace,
50
+ 'consumptionMethod': self.consumption_method
51
+ }
52
+
53
+
54
+ class Toleration:
55
+ def __init__(self, effect: str, key: str, operator: str, value: str):
56
+ self.effect = effect
57
+ self.key = key
58
+ self.operator = operator
59
+ self.value = value
60
+
61
+ @classmethod
62
+ def from_json(cls, _json):
63
+ return cls(
64
+ effect=_json.get('effect'),
65
+ key=_json.get('key'),
66
+ operator=_json.get('operator'),
67
+ value=_json.get('value')
68
+ )
69
+
70
+ def to_json(self):
71
+ return {
72
+ 'effect': self.effect,
73
+ 'key': self.key,
74
+ 'operator': self.operator,
75
+ 'value': self.value
76
+ }
77
+
78
+
79
+ class DeploymentResource:
80
+ def __init__(self, gpu: int, cpu: int, memory: str):
81
+ self.gpu = gpu
82
+ self.cpu = cpu
83
+ self.memory = memory
84
+
85
+ @classmethod
86
+ def from_json(cls, _json):
87
+ return cls(
88
+ gpu=_json.get('gpu', None),
89
+ cpu=_json.get('cpu', None),
90
+ memory=_json.get('memory', None)
91
+ )
92
+
93
+ def to_json(self):
94
+ _json = {}
95
+ if self.gpu is not None:
96
+ _json['gpu'] = self.gpu
97
+ if self.cpu is not None:
98
+ _json['cpu'] = self.cpu
99
+ if self.memory is not None:
100
+ _json['memory'] = self.memory
101
+
102
+
103
+ class DeploymentResources:
104
+ def __init__(self, request: DeploymentResource, limit: DeploymentResource):
105
+ self.request = request
106
+ self.limit = limit
107
+
108
+ @classmethod
109
+ def from_json(cls, _json):
110
+ return cls(
111
+ request=DeploymentResource.from_json(_json.get('request') or dict()),
112
+ limit=DeploymentResource.from_json(_json.get('limit') or dict())
113
+ )
114
+
115
+ def to_json(self):
116
+ return {
117
+ 'request': self.request.to_json(),
118
+ 'limit': self.limit.to_json()
119
+ }
120
+
121
+
122
+ class NodePool:
123
+ def __init__(
124
+ self,
125
+ name: str,
126
+ is_dl_type_default: bool,
127
+ dl_types: Optional[List[str]] = None,
128
+ tolerations: Optional[List[Toleration]] = None,
129
+ description: str = "",
130
+ node_selector: str = "",
131
+ preemptible: bool = False,
132
+ deployment_resources: DeploymentResources = None
133
+ ):
134
+ self.name = name
135
+ self.is_dl_type_default = is_dl_type_default
136
+ self.dl_types = dl_types
137
+ self.tolerations = tolerations if tolerations is not None else []
138
+ self.description = description
139
+ self.node_selector = node_selector
140
+ self.preemptible = preemptible
141
+ self.deployment_resources = deployment_resources
142
+
143
+ @classmethod
144
+ def from_json(cls, _json):
145
+ node_pool = cls(
146
+ name=_json.get('name'),
147
+ is_dl_type_default=_json.get('isDlTypeDefault'),
148
+ dl_types=_json.get('dlTypes'),
149
+ description=_json.get('description'),
150
+ node_selector=_json.get('nodeSelector'),
151
+ preemptible=_json.get('preemptible'),
152
+ deployment_resources=DeploymentResources.from_json(_json.get('deploymentResources', dict())),
153
+ tolerations=[Toleration.from_json(t) for t in _json.get('tolerations', list())]
154
+ )
155
+
156
+ return node_pool
157
+
158
+ def to_json(self):
159
+ _json = {
160
+ 'name': self.name,
161
+ 'isDlTypeDefault': self.is_dl_type_default,
162
+ 'description': self.description,
163
+ 'nodeSelector': self.node_selector,
164
+ 'preemptible': self.preemptible,
165
+ 'deploymentResources': self.deployment_resources.to_json(),
166
+ 'tolerations': [t.to_json() for t in self.tolerations]
167
+ }
168
+
169
+ if self.dl_types is not None:
170
+ _json['dlTypes'] = self.dl_types
171
+
172
+ return _json
173
+
174
+
175
+ class AuthenticationIntegration:
176
+ def __init__(self, id: str, type: str):
177
+ self.id = id
178
+ self.type = type
179
+
180
+ @classmethod
181
+ def from_json(cls, _json):
182
+ return cls(
183
+ id=_json.get('id'),
184
+ type=_json.get('type')
185
+ )
186
+
187
+ def to_json(self):
188
+ return {
189
+ 'id': self.id,
190
+ 'type': self.type
191
+ }
192
+
193
+
194
+ class Authentication:
195
+ def __init__(self, integration: AuthenticationIntegration):
196
+ self.integration = integration
197
+
198
+ @classmethod
199
+ def from_json(cls, _json):
200
+ return cls(
201
+ integration=AuthenticationIntegration.from_json(_json.get('integration', dict()))
202
+ )
203
+
204
+ def to_json(self):
205
+ return {
206
+ 'integration': self.integration.to_json()
207
+ }
208
+
209
+
210
+ class ComputeCluster:
211
+ def __init__(
212
+ self,
213
+ name: str,
214
+ endpoint: str,
215
+ kubernetes_version: str,
216
+ provider: ClusterProvider,
217
+ node_pools: Optional[List[NodePool]] = None,
218
+ metadata: Optional[Dict] = None,
219
+ authentication: Optional[Authentication] = None,
220
+ plugins: Optional[dict] = None
221
+ ):
222
+ self.name = name
223
+ self.endpoint = endpoint
224
+ self.kubernetes_version = kubernetes_version
225
+ self.provider = provider
226
+ self.node_pools = node_pools if node_pools is not None else []
227
+ self.metadata = metadata if metadata is not None else {}
228
+ self.authentication = authentication if authentication is not None else Authentication(
229
+ AuthenticationIntegration("", ""))
230
+ self.plugins = plugins
231
+
232
+ @classmethod
233
+ def from_json(cls, _json):
234
+ return cls(
235
+ name=_json.get('name'),
236
+ endpoint=_json.get('endpoint'),
237
+ kubernetes_version=_json.get('kubernetesVersion'),
238
+ provider=ClusterProvider(_json.get('provider')),
239
+ node_pools=[NodePool.from_json(np) for np in _json.get('nodePools', list())],
240
+ metadata=_json.get('metadata'),
241
+ authentication=Authentication.from_json(_json.get('authentication', dict())),
242
+ plugins=_json.get('plugins')
243
+ )
244
+
245
+ def to_json(self):
246
+ return {
247
+ 'name': self.name,
248
+ 'endpoint': self.endpoint,
249
+ 'kubernetesVersion': self.kubernetes_version,
250
+ 'provider': self.provider.value,
251
+ 'nodePools': [np.to_json() for np in self.node_pools],
252
+ 'metadata': self.metadata,
253
+ 'authentication': self.authentication.to_json(),
254
+ 'plugins': self.plugins
255
+ }
256
+
257
+ @classmethod
258
+ def from_setup_json(cls, devops_output, integration):
259
+ node_pools = [NodePool.from_json(n) for n in devops_output['config']['nodePools']]
260
+ return cls(
261
+ name=devops_output['config']['name'],
262
+ endpoint=devops_output['config']['endpoint'],
263
+ kubernetes_version=devops_output['config']['kubernetesVersion'],
264
+ provider=ClusterProvider(devops_output['config']['provider']),
265
+ node_pools=node_pools,
266
+ metadata={},
267
+ authentication=Authentication(AuthenticationIntegration(integration.id, integration.type)),
268
+ plugins=devops_output['config'].get('plugins')
269
+ )
270
+
271
+
272
+ class ComputeContext:
273
+ def __init__(self, labels: List[str], org: str, project: Optional[str] = None):
274
+ self.labels = labels
275
+ self.org = org
276
+ self.project = project
277
+
278
+ @classmethod
279
+ def from_json(cls, _json):
280
+ return cls(
281
+ labels=_json.get('labels', list()),
282
+ org=_json.get('org'),
283
+ project=_json.get('project')
284
+ )
285
+
286
+ def to_json(self):
287
+ return {
288
+ 'labels': self.labels,
289
+ 'org': self.org,
290
+ 'project': self.project
291
+ }
292
+
293
+
294
+ class Compute:
295
+ def __init__(
296
+ self,
297
+ id: str,
298
+ name: str,
299
+ context: ComputeContext,
300
+ client_api: ApiClient,
301
+ shared_contexts: Optional[List[ComputeContext]] = None,
302
+ global_: Optional[bool] = None,
303
+ status: ComputeStatus = ComputeStatus.INITIALIZING,
304
+ type: ComputeType = ComputeType.KUBERNETES,
305
+ features: Optional[Dict] = None,
306
+ metadata: Optional[Dict] = None,
307
+ settings: Optional[ComputeSettings] = None,
308
+ url: Optional[str] = None
309
+ ):
310
+ self.id = id
311
+ self.name = name
312
+ self.context = context
313
+ self.shared_contexts = shared_contexts if shared_contexts is not None else []
314
+ self.global_ = global_
315
+ self.status = status
316
+ self.type = type
317
+ self.features = features if features is not None else dict()
318
+ self.metadata = metadata if metadata is not None else dict()
319
+ self._client_api = client_api
320
+ self._computes = None
321
+ self._serviceDrivers = None
322
+ self.settings = settings
323
+ self.url = url
324
+
325
+ @property
326
+ def computes(self):
327
+ if self._computes is None:
328
+ self._computes = repositories.Computes(client_api=self._client_api)
329
+ return self._computes
330
+
331
+ def delete(self, skip_destroy: bool = False):
332
+ return self.computes.delete(compute_id=self.id, skip_destroy=skip_destroy)
333
+
334
+ def update(self):
335
+ return self.computes.update(compute=self)
336
+
337
+ @staticmethod
338
+ def _protected_from_json(_json: dict, client_api: ApiClient):
339
+ """
340
+ Same as from_json but with try-except to catch if error
341
+
342
+ :param _json: platform json
343
+ :param client_api: ApiClient entity
344
+ :return:
345
+ """
346
+ try:
347
+ compute = Compute.from_json(_json=_json,
348
+ client_api=client_api)
349
+ status = True
350
+ except Exception:
351
+ compute = traceback.format_exc()
352
+ status = False
353
+ return status, compute
354
+
355
+ @classmethod
356
+ def from_json(cls, _json, client_api: ApiClient):
357
+ return cls(
358
+ id=_json.get('id'),
359
+ name=_json.get('name'),
360
+ context=ComputeContext.from_json(_json.get('context', dict())),
361
+ shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
362
+ global_=_json.get('global'),
363
+ status=ComputeStatus(_json.get('status')),
364
+ type=ComputeType(_json.get('type')),
365
+ features=_json.get('features'),
366
+ client_api=client_api,
367
+ metadata=_json.get('metadata'),
368
+ settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
369
+ url=_json.get('url'),
370
+ )
371
+
372
+ def to_json(self):
373
+ return {
374
+ 'id': self.id,
375
+ 'name': self.name,
376
+ 'context': self.context.to_json(),
377
+ 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
378
+ 'global': self.global_,
379
+ 'status': self.status.value,
380
+ 'type': self.type.value,
381
+ 'features': self.features,
382
+ 'metadata': self.metadata,
383
+ 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
384
+ 'url': self.url
385
+ }
386
+
387
+
388
+ class KubernetesCompute(Compute):
389
+ def __init__(
390
+ self,
391
+ id: str,
392
+ name: str,
393
+ context: ComputeContext,
394
+ cluster: ComputeCluster,
395
+ shared_contexts: Optional[List[ComputeContext]] = None,
396
+ global_: Optional[bool] = None,
397
+ status: ComputeStatus = ComputeStatus.INITIALIZING,
398
+ type: ComputeType = ComputeType.KUBERNETES,
399
+ features: Optional[Dict] = None,
400
+ metadata: Optional[Dict] = None,
401
+ client_api: ApiClient = None,
402
+ settings: Optional[ComputeSettings] = None,
403
+ url: Optional[str] = None
404
+ ):
405
+ super().__init__(id=id, context=context, shared_contexts=shared_contexts, global_=global_, status=status,
406
+ type=type, features=features, metadata=metadata, client_api=client_api, settings=settings,
407
+ name=name, url=url)
408
+ self.cluster = cluster
409
+
410
+ @classmethod
411
+ def from_json(cls, _json, client_api: ApiClient):
412
+ return cls(
413
+ id=_json.get('id'),
414
+ name=_json.get('name'),
415
+ context=ComputeContext.from_json(_json.get('context', dict())),
416
+ cluster=ComputeCluster.from_json(_json.get('cluster', dict())),
417
+ shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
418
+ global_=_json.get('global'),
419
+ status=ComputeStatus(_json.get('status')),
420
+ type=ComputeType(_json.get('type')),
421
+ features=_json.get('features'),
422
+ metadata=_json.get('metadata'),
423
+ client_api=client_api,
424
+ settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
425
+ url=_json.get('url'),
426
+ )
427
+
428
+ def to_json(self):
429
+ return {
430
+ 'id': self.id,
431
+ 'name': self.name,
432
+ 'context': self.context.to_json(),
433
+ 'cluster': self.cluster.to_json(),
434
+ 'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
435
+ 'global': self.global_,
436
+ 'status': self.status.value,
437
+ 'type': self.type.value,
438
+ 'features': self.features,
439
+ 'metadata': self.metadata,
440
+ 'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
441
+ 'url': self.url
442
+ }