dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,764 +1,764 @@
1
- import subprocess
2
- import inquirer
3
- import logging
4
- import json
5
- import os
6
- import sys
7
- import jwt
8
-
9
- from .. import exceptions, entities, repositories, utilities, assets
10
-
11
- logger = logging.getLogger(name='dtlpy')
12
-
13
-
14
- class CommandExecutor:
15
-
16
- def __init__(self, dl, parser):
17
- self.dl = dl
18
- self.parser = parser
19
- self.utils = Utils(dl)
20
-
21
- def run(self, args):
22
- ########################
23
- # Run Command if Exist #
24
- ########################
25
- if args.operation is None:
26
- logger.info('See "dlp --help" for options')
27
- return
28
-
29
- operation = args.operation.lower().replace('-', '_')
30
- if hasattr(self, operation):
31
- getattr(self, operation)(args)
32
- ###############
33
- # Catch typos #
34
- ###############
35
- elif args.operation in ["project", 'dataset', 'item', 'service', 'package', 'video', 'deploy', 'generate']:
36
- self.typos(args=args)
37
- #######################
38
- # Catch other options #
39
- #######################
40
- elif args.operation:
41
- print('dlp: "%s" is not an dlp command' % args.operation)
42
- print('See "dlp --help" for options')
43
- else:
44
- print('See "dlp --help" for options')
45
-
46
- def help(self, args):
47
- self.parser.print_help()
48
-
49
- def logout(self, args):
50
- self.dl.logout()
51
- logger.info('logout successful')
52
-
53
- # noinspection PyUnusedLocal
54
- def login(self, args):
55
- self.dl.login()
56
- self.dl.info(with_token=False)
57
-
58
- def login_token(self, args):
59
- self.dl.login_token(args.token)
60
- self.dl.info(with_token=False)
61
-
62
- def login_api_key(self, args):
63
- self.dl.login_api_key(api_key=args.api_key)
64
-
65
- def login_secret(self, args):
66
- self.login_m2m(args=args)
67
-
68
- def login_m2m(self, args):
69
- self.dl.login_m2m(email=args.email,
70
- password=args.password,
71
- client_id=args.client_id,
72
- client_secret=args.client_secret)
73
- self.dl.info(with_token=False)
74
-
75
- def upgrade(self, args):
76
- url = 'dtlpy'
77
- if args.url is None:
78
- try:
79
- payload = jwt.decode(self.dl.client_api.token, algorithms=['HS256'],
80
- verify=False, options={'verify_signature': False})
81
- if 'admin' in payload['https://dataloop.ai/authorization']['roles']:
82
- url = "https://storage.googleapis.com/dtlpy/dev/dtlpy-latest-py3-none-any.whl"
83
- except Exception:
84
- pass
85
- else:
86
- url = args.url
87
-
88
- logger.info("Update DTLPy from {}".format(url))
89
- logger.info("Installing using pip...")
90
- cmd = "pip install {} --upgrade".format(url)
91
- subprocess.Popen(cmd, shell=True)
92
- sys.exit(0)
93
-
94
- # noinspection PyUnusedLocal
95
- def init(self, args):
96
- self.dl.init()
97
-
98
- # noinspection PyUnusedLocal
99
- def checkout_state(self, args):
100
- state = self.dl.checkout_state()
101
- logger.info('Checked-out:')
102
- for key, val in state.items():
103
- try:
104
- msg = '{entity} name: {name}\t{entity} id: {id}'.format(entity=key, name=val['name'], id=val['id'])
105
- except KeyError:
106
- msg = '{entity} Not Found'.format(entity=key)
107
- logger.info(msg)
108
-
109
- # noinspection PyUnusedLocal
110
- def version(self, args):
111
- logger.info("Dataloop SDK Version: {}".format(self.dl.__version__))
112
-
113
- def development(self, args):
114
- if args.development == "local":
115
- # start the local development
116
- if args.local == 'start':
117
- development = self.utils.dl.client_api.state_io.get('development')
118
- # create default values
119
- if development is None:
120
- development = dict()
121
- if 'port' not in development:
122
- development['port'] = 5802
123
- if 'docker_image' not in development:
124
- development['docker_image'] = 'dataloopai/dtlpy-agent:1.57.3.gpu.cuda11.5.py3.8.opencv'
125
-
126
- # get values from input
127
- if args.docker_image is not None:
128
- development['docker_image'] = args.docker_image
129
- if args.port is not None:
130
- development['port'] = int(args.port)
131
-
132
- # set values to local state
133
- self.utils.dl.client_api.state_io.put('development', development)
134
- utilities.local_development.start_session()
135
- elif args.local == 'pause':
136
- utilities.local_development.pause_session()
137
- elif args.local == 'stop':
138
- utilities.local_development.stop_session()
139
- else:
140
- print('Must select one of "start", "pause", "stop". Type "dlp development start --help" for options')
141
- elif args.development == "remote":
142
- logger.warning('FUTURE! This is not supported yet..')
143
- else:
144
- print('Type "dlp development --help" for options')
145
-
146
- # noinspection PyUnusedLocal
147
-
148
- def api(self, args):
149
- if args.api == "info":
150
- information = self.dl.info()
151
- logger.info('-- Dataloop info --')
152
- _ = [logger.info('{}: {}'.format(key, val)) for key, val in information.items()]
153
-
154
- if args.api == "setenv":
155
- self.dl.setenv(args.env)
156
- logger.info("Platform environment: {}".format(self.dl.environment()))
157
-
158
- def projects(self, args):
159
- if args.projects == "ls":
160
- self.dl.projects.list().print()
161
- elif args.projects == "web":
162
- if args.project_name is None:
163
- args.project_name = self.dl.projects.get().name
164
- self.dl.projects.open_in_web(project_name=args.project_name)
165
- elif args.projects == "create":
166
- project = self.dl.projects.create(args.project_name)
167
- project.print()
168
- project.checkout()
169
- elif args.projects == "checkout":
170
- self.dl.projects.checkout(project_name=args.project_name)
171
- else:
172
- print('Type "dlp projects --help" for options')
173
-
174
- def datasets(self, args):
175
- if args.datasets == "ls":
176
- self.utils.get_datasets_repo(args=args).list().print()
177
-
178
- elif args.datasets == "checkout":
179
- self.dl.datasets.checkout(dataset_name=args.dataset_name)
180
-
181
- elif args.datasets == "web":
182
- if args.dataset_name is None:
183
- args.dataset_name = self.dl.datasets.get().name
184
- self.utils.get_datasets_repo(args=args).open_in_web(dataset_name=args.dataset_name)
185
-
186
- elif args.datasets == "create":
187
- dataset = self.utils.get_datasets_repo(args=args).create(dataset_name=args.dataset_name)
188
- dataset.print()
189
- if args.checkout:
190
- dataset.checkout()
191
- else:
192
- print('Type "dlp datasets --help" for options')
193
-
194
- def items(self, args):
195
- if self.dl.token_expired():
196
- logger.error("token expired, please login.")
197
- return
198
-
199
- if args.items == "ls":
200
- project = self.dl.projects.get(project_name=args.project_name)
201
- dataset = project.datasets.get(dataset_name=args.dataset_name)
202
- if isinstance(args.page, str):
203
- try:
204
- args.page = int(args.page)
205
- except ValueError:
206
- raise ValueError("Input --page must be integer")
207
- filters = self.dl.Filters()
208
-
209
- # add filters
210
- if args.remote_path is not None:
211
- if isinstance(args.remote_path, list):
212
- filters.add(field="filename", values=args.remote_path, operator=entities.FiltersOperations.IN)
213
- else:
214
- filters.add(field="filename", values=args.remote_path)
215
- if args.type is not None:
216
- if isinstance(args.type, list):
217
- filters.add(field='metadata.system.mimetype', values=args.type,
218
- operator=entities.FiltersOperations.IN)
219
- else:
220
- filters.add(field="metadata.system.mimetype", values=args.type)
221
-
222
- pages = dataset.items.list(filters=filters, page_offset=args.page)
223
- pages.print()
224
- print("Displaying page %d/%d" % (args.page, pages.total_pages_count - 1))
225
-
226
- elif args.items == "web":
227
- project = self.dl.projects.get(project_name=args.project_name)
228
- dataset = project.datasets.get(dataset_name=args.dataset_name)
229
- dataset.items.open_in_web(filepath=args.remote_path)
230
-
231
- elif args.items == "upload":
232
- logger.info("Uploading directory...")
233
- if isinstance(args.file_types, str):
234
- args.file_types = [t.strip() for t in args.file_types.split(",")]
235
- project = self.dl.projects.get(project_name=args.project_name)
236
- dataset = project.datasets.get(dataset_name=args.dataset_name)
237
-
238
- dataset.items.upload(local_path=args.local_path,
239
- remote_path=args.remote_path,
240
- file_types=args.file_types,
241
- overwrite=args.overwrite,
242
- local_annotations_path=args.local_annotations_path)
243
-
244
- elif args.items == "download":
245
- logger.info("Downloading dataset...")
246
- project = self.dl.projects.get(project_name=args.project_name)
247
- dataset = project.datasets.get(dataset_name=args.dataset_name)
248
-
249
- annotation_options = None
250
- if args.annotation_options is not None:
251
- annotation_options = [t.strip() for t in args.annotation_options.split(",")]
252
-
253
- annotation_filters = None
254
- if args.annotation_filter_type is not None or args.annotation_filter_label is not None:
255
- annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
256
- if args.annotation_filter_type is not None:
257
- annotation_filter_type = [t.strip() for t in args.annotation_filter_type.split(",")]
258
- annotation_filters.add(field='type',
259
- values=annotation_filter_type,
260
- operator=entities.FiltersOperations.IN)
261
- if args.annotation_filter_label is not None:
262
- annotation_filter_label = [t.strip() for t in args.annotation_filter_label.split(",")]
263
- annotation_filters.add(field='label',
264
- values=annotation_filter_label,
265
- operator=entities.FiltersOperations.IN)
266
- # create remote path filters
267
- filters = self.dl.Filters()
268
- if args.remote_path is not None:
269
- remote_path = [t.strip() for t in args.remote_path.split(",")]
270
- if len(remote_path) == 1:
271
- remote_path = remote_path[0]
272
- filters.add(field="filename", values=remote_path)
273
- elif len(remote_path) > 1:
274
- for item in remote_path:
275
- if '*' in item:
276
- filters.add(field="dir", values=item, method='or')
277
- remote_path.pop(remote_path.index(item))
278
- filters.add(field="dir", values=remote_path, operator=entities.FiltersOperations.IN, method='or')
279
-
280
- if not args.without_binaries:
281
- dataset.items.download(filters=filters,
282
- local_path=args.local_path,
283
- annotation_options=annotation_options,
284
- annotation_filters=annotation_filters,
285
- overwrite=args.overwrite,
286
- with_text=args.with_text,
287
- thickness=int(args.thickness),
288
- to_items_folder=not args.not_items_folder)
289
- else:
290
- dataset.download_annotations(filters=filters,
291
- local_path=args.local_path,
292
- annotation_options=annotation_options,
293
- annotation_filters=annotation_filters,
294
- overwrite=args.overwrite,
295
- with_text=args.with_text,
296
- thickness=int(args.thickness))
297
-
298
- else:
299
- print('Type "dlp items --help" for options')
300
-
301
- def videos(self, args):
302
- if self.dl.token_expired():
303
- print("[ERROR] token expired, please login.")
304
- return
305
-
306
- if args.videos == "play":
307
- from dtlpy.utilities.videos.video_player import VideoPlayer
308
-
309
- VideoPlayer(
310
- item_filepath=args.item_path,
311
- dataset_name=args.dataset_name,
312
- project_name=args.project_name,
313
- )
314
-
315
- elif args.videos == "upload":
316
- if (
317
- (args.split_chunks is not None)
318
- or (args.split_seconds is not None)
319
- or (args.split_times is not None)
320
- ):
321
- # upload with split
322
- if isinstance(args.split_chunks, str):
323
- args.split_chunks = int(args.split_chunks)
324
- if isinstance(args.split_seconds, str):
325
- args.split_seconds = int(args.split_seconds)
326
- if isinstance(args.split_times, str):
327
- args.split_times = [int(sec) for sec in args.split_times.split(",")]
328
- self.dl.utilities.videos.Videos.split_and_upload(
329
- project_name=args.project_name,
330
- dataset_name=args.dataset_name,
331
- filepath=args.filename,
332
- remote_path=args.remote_path,
333
- split_chunks=args.split_chunks,
334
- split_seconds=args.split_seconds,
335
- split_pairs=args.split_times,
336
- )
337
- else:
338
- print('Type "dlp videos --help" for options')
339
-
340
- def services(self, args):
341
- if self.dl.token_expired():
342
- print("[ERROR] token expired, please login.")
343
- return
344
-
345
- elif args.services == "delete":
346
- service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
347
- service.delete()
348
- logger.info('Service: "{}" deleted successfully'.format(service.name))
349
-
350
- elif args.services == "ls":
351
- self.utils.get_services_repo(args=args).list().print()
352
- elif args.services == "log":
353
- project = self.dl.projects.get(project_name=args.project_name)
354
- service = project.services.get(service_name=args.service_name)
355
-
356
- logs = service.log(start=args.start)
357
- try:
358
- for log in logs:
359
- if isinstance(log, list):
360
- for log_record in log:
361
- print(log_record)
362
- else:
363
- print(log)
364
- except KeyboardInterrupt:
365
- pass
366
-
367
- elif args.services == "execute":
368
- try:
369
- if args.service_name:
370
- service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
371
- else:
372
- service = self.dl.services.get()
373
- except Exception:
374
- logger.info('Please checkout a service')
375
- return
376
-
377
- if args.annotation_id is not None:
378
- resource = 'Annotation'
379
- elif args.item_id is not None:
380
- resource = 'Item'
381
- elif args.dataset_id is not None:
382
- resource = 'Dataset'
383
- else:
384
- resource = None
385
-
386
- try:
387
- execution_input = json.loads(args.inputs)
388
- except Exception:
389
- logger.info('Input should be json serializable')
390
- return
391
- if len(execution_input) == 0:
392
- execution_input = None
393
-
394
- logger.info('executing')
395
- service.execute(sync=not args.asynchronous,
396
- execution_input=execution_input,
397
- function_name=args.function_name,
398
- resource=resource,
399
- item_id=args.item_id,
400
- dataset_id=args.dataset_id,
401
- annotation_id=args.annotation_id)
402
- logger.info("Successfully executed function: {}".format(args.function_name))
403
-
404
- else:
405
- logger.info('Type "dlp packages --help" for options')
406
-
407
- def deploy(self, args):
408
- project = self.dl.projects.get(project_name=args.project_name)
409
- json_filepath = args.json_file
410
- deployed_services, package = self.dl.packages.deploy_from_file(project=project, json_filepath=json_filepath)
411
- logger.info("Successfully deployed {} from file: {}\nServices: {}".format(len(deployed_services),
412
- json_filepath,
413
- [s.name for s in deployed_services]))
414
-
415
- def generate(self, args):
416
- package_type = args.package_type if args.package_type else self.dl.PackageCatalog.DEFAULT_PACKAGE_TYPE
417
- self.dl.packages.generate(name=args.package_name, src_path=os.getcwd(), package_type=package_type)
418
- self.utils.dl.client_api.state_io.put('package', {'name': args.package_name})
419
- logger.info('Successfully generated package files')
420
-
421
- def triggers(self, args):
422
-
423
- if args.triggers == "create":
424
- args.actions = [t.strip() for t in args.actions.split(",")]
425
- try:
426
- service = self.utils.get_services_repo(args).get(service_name=args.service_name)
427
- except exceptions.NotFound:
428
- logger.critical('Service not found. Please check-out a service or provide valid service name')
429
- return
430
-
431
- trigger = service.triggers.create(name=args.name,
432
- filters=json.loads('{}'.format(args.filters.replace("'", '"'))),
433
- function_name=args.function_name,
434
- resource=args.resource,
435
- actions=args.actions)
436
- logger.info('Trigger created successfully: {}'.format(trigger.name))
437
-
438
- elif args.triggers == "delete":
439
- triggers = self.utils.get_triggers_repo(args=args)
440
- triggers.get(trigger_name=args.trigger_name).delete()
441
- logger.info('Trigger deleted successfully: {}'.format(args.trigger_name))
442
- elif args.triggers == "ls":
443
- triggers = self.utils.get_triggers_repo(args=args)
444
- triggers.list().print()
445
- else:
446
- logger.info('Type "dlp packages --help" for options')
447
-
448
- def packages(self, args):
449
- if self.dl.token_expired():
450
- logger.error("token expired, please login.")
451
- return
452
-
453
- elif args.packages == "delete":
454
- package = self.utils.get_packages_repo(args=args).get(package_name=args.package_name)
455
- package.delete()
456
- logger.info('Successfully deleted package {}'.format(package.name))
457
-
458
- elif args.packages == "ls":
459
- self.utils.get_packages_repo(args=args).list().print()
460
-
461
- elif args.packages == "checkout":
462
- self.dl.packages.checkout(package_name=args.package_name)
463
-
464
- elif args.packages == "push":
465
- packages = self.utils.get_packages_repo(args=args)
466
-
467
- package = packages.push(src_path=args.src_path,
468
- package_name=args.package_name,
469
- checkout=args.checkout)
470
-
471
- logger.info("Successfully pushed package to platform\n"
472
- "Package id:{}\nPackage version:{}".format(package.id,
473
- package.version))
474
- elif args.packages == "deploy":
475
- packages = self.utils.get_packages_repo(args=args)
476
-
477
- package = packages.deploy(package_name=args.package_name,
478
- checkout=args.checkout,
479
- module_name=args.module_name)
480
-
481
- logger.info("Successfully pushed package to platform\n"
482
- "Package id:{}\nPackage version:{}".format(package.id,
483
- package.version))
484
-
485
- elif args.packages == "test":
486
- go_back = False
487
- if 'src' in os.listdir(os.getcwd()):
488
- go_back = True
489
- os.chdir('./src')
490
- try:
491
- self.dl.packages.test_local_package(concurrency=int(args.concurrency),
492
- function_name=args.function_name)
493
- except Exception:
494
- logger.exception('failed during test')
495
- finally:
496
- if go_back:
497
- os.chdir('..')
498
-
499
- else:
500
- logger.info('Type "dlp packages --help" for options')
501
-
502
- def app(self, args):
503
- if args.app == 'pack':
504
- path = self.dl.dpks.pack()
505
- logger.info(f'Packed to {path}')
506
- elif args.app == 'init':
507
- app_filename = assets.paths.APP_JSON_FILENAME
508
- if os.path.isfile(app_filename):
509
- questions = [
510
- inquirer.Confirm(name='overwrite',
511
- message=f"Dataloop app already initialized. Re-initialize?",
512
- default=False)]
513
- answers = inquirer.prompt(questions)
514
- if answers.get('overwrite') is False:
515
- return
516
- name = args.name
517
- description = args.description
518
- attributes = args.attributes
519
- icon = args.icon
520
- scope = args.scope
521
- as_array = [name, description, attributes, icon, scope]
522
- if as_array.count(None) == len(as_array): # No one value is initialized
523
- dir_name = os.path.basename(os.getcwd())
524
- questions = [
525
- inquirer.Text(name='name',
526
- message=f"Enter the name of the app (or press enter for '{dir_name}'):",
527
- default=dir_name),
528
- inquirer.Text(name='description',
529
- message="Enter the description (or enter for empty): "),
530
- inquirer.Text(name='attributes',
531
- message="Enter the attributes (an object, or enter for empty): ",
532
- default=None),
533
- inquirer.Text(name='icon',
534
- message="Enter the path to the icon (or enter for empty): "),
535
- inquirer.Text(name='scope',
536
- message="Enter the scope (or enter for 'organization'): ",
537
- default='project'),
538
- ]
539
- answers = inquirer.prompt(questions)
540
- name = answers.get('name')
541
- description = answers.get('description')
542
- attributes = answers.get('attributes')
543
- icon = answers.get('icon')
544
- scope = answers.get('scope')
545
-
546
- if attributes is not None:
547
- attributes = json.loads(attributes)
548
-
549
- self.dl.dpks.init(name=name,
550
- description=description,
551
- attributes=attributes,
552
- icon=icon,
553
- scope=scope)
554
- elif args.app == 'add':
555
- if args.panel is True:
556
- default_panel_name = "myPanel"
557
- choices = list(entities.dpk.SlotType)
558
-
559
- questions = [
560
- inquirer.Text(name='name',
561
- message=f"Enter PANEL NAME (or press enter for '{default_panel_name}'): ",
562
- default='default_panel_name'),
563
- inquirer.List(name='support_slot_type',
564
- message="Enter SUPPORTED SLOT TYPE:",
565
- choices=choices),
566
- ]
567
-
568
- answers = inquirer.prompt(questions)
569
- #####
570
- # create a dir for that panel
571
- os.makedirs(answers.get('name'), exist_ok=True)
572
- # dump to dataloop.json
573
- app_filename = assets.paths.APP_JSON_FILENAME
574
- if not os.path.isfile(app_filename):
575
- logger.error(f"Can't find app config file ({app_filename}), please run `dlp app init` first")
576
- else:
577
- with open(app_filename, 'r') as f:
578
- dpk = entities.Dpk.from_json(json.load(f))
579
- dpk.components.panels.append(entities.Panel(name=answers.get('name'),
580
- supported_slots=[answers.get('support_slot_type')]))
581
- with open(app_filename, 'w') as f:
582
- json.dump(dpk.to_json(), f, indent=4)
583
-
584
- elif args.app == 'publish':
585
- dpk = self.utils.get_dpks_repo(args).publish()
586
- if dpk:
587
- logger.info(f'Published the application: id={dpk.id}')
588
- else:
589
- logger.info("Couldn't publish the application")
590
- elif args.app == 'update':
591
- # TODO: I think it changed, not implemented
592
- logger.info('App updated successfully')
593
- elif args.app == 'resume':
594
- succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
595
- if succeed is True:
596
- logger.info('Resumed application successfully')
597
- else:
598
- logger.info('Application resume failed')
599
- elif args.app == 'pause':
600
- succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
601
- if succeed is True:
602
- logger.info('Paused application successfully')
603
- else:
604
- logger.info('Application pause failed')
605
- elif args.app == 'install':
606
- app = self.utils.get_apps_repo(args).install(
607
- dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
608
- organization_id=args.org_id
609
- )
610
- if app is not None:
611
- logger.info('App installed successfully')
612
- else:
613
- logger.info("Couldn't install the app")
614
- elif args.app == 'pull':
615
- succeed = self.dl.dpks.pull(dpk_name=args.dpk_name)
616
- if succeed is True:
617
- logger.info("Pulled successfully")
618
- else:
619
- logger.info("Couldn't pull")
620
- elif args.app == 'list':
621
- self.dl.apps.list().print()
622
-
623
- # noinspection PyUnusedLocal
624
- @staticmethod
625
- def pwd(args):
626
- print(os.getcwd())
627
-
628
- @staticmethod
629
- def cd(args):
630
- directory = args.dir
631
- if directory == '..':
632
- directory = os.path.split(os.getcwd())[0]
633
- os.chdir(directory)
634
- print(os.getcwd())
635
-
636
- @staticmethod
637
- def mkdir(args):
638
- os.mkdir(args.name)
639
-
640
- # noinspection PyUnusedLocal
641
- @staticmethod
642
- def ls(args):
643
- print(os.getcwd())
644
- dirs = os.listdir(os.getcwd())
645
- import pprint
646
-
647
- pp = pprint.PrettyPrinter(indent=3)
648
- pp.pprint(dirs)
649
-
650
- # noinspection PyUnusedLocal
651
- @staticmethod
652
- def clear(args):
653
- if os.name == "nt":
654
- os.system('cls')
655
- else:
656
- os.system('clear')
657
-
658
- @staticmethod
659
- def typos(args):
660
- print('dlp: "{op}" is not an dlp command. Did you mean "{op}s"?'.format(op=args.operation))
661
-
662
-
663
- class Utils:
664
- def __init__(self, dl):
665
- self.dl = dl
666
-
667
- def get_packages_repo(self, args):
668
- if args.project_name is not None:
669
- project = self.dl.projects.get(project_name=args.project_name)
670
- packages = project.packages
671
- else:
672
- try:
673
- packages = self.dl.projects.get().packages
674
- except Exception:
675
- packages = self.dl.packages
676
-
677
- assert isinstance(packages, repositories.Packages)
678
- return packages
679
-
680
- def get_datasets_repo(self, args):
681
- if args.project_name is not None:
682
- project = self.dl.projects.get(project_name=args.project_name)
683
- datasets = project.datasets
684
- else:
685
- try:
686
- datasets = self.dl.projects.get().datasets
687
- except Exception:
688
- datasets = self.dl.datasets
689
-
690
- assert isinstance(datasets, repositories.Datasets)
691
- return datasets
692
-
693
- def get_services_repo(self, args):
694
- if args.project_name is not None:
695
- project = self.dl.projects.get(project_name=args.project_name)
696
- packages = project.packages
697
- services = project.services
698
- else:
699
- try:
700
- packages = self.dl.projects.get().packages
701
- services = self.dl.projects.get().services
702
- except Exception:
703
- packages = self.dl.packages
704
- services = self.dl.services
705
-
706
- if args.package_name is not None:
707
- package = packages.get(package_name=args.package_name)
708
- services = package.services
709
-
710
- assert isinstance(services, repositories.Services)
711
- return services
712
-
713
- def get_apps_repo(self, args) -> repositories.Apps:
714
- if args.project_name is not None:
715
- project = self.dl.projects.get(project_name=args.project_name)
716
- apps = project.apps
717
- else:
718
- try:
719
- apps = self.dl.projects.get().apps
720
- except Exception:
721
- apps = self.dl.apps
722
- assert isinstance(apps, repositories.Apps)
723
- return apps
724
-
725
- def get_dpks_repo(self, args) -> repositories.Dpks:
726
- if args.project_name is None and args.project_id is None:
727
- try:
728
- dpks = self.dl.projects.get().dpks
729
- except Exception:
730
- dpks = self.dl.dpks
731
- else:
732
- project = self.dl.projects.get(project_name=args.project_name, project_id=args.project_id)
733
- dpks = project.dpks
734
- if dpks.project is None:
735
- raise ValueError("Must input one of `project-name` or `project-id`") from None
736
- assert isinstance(dpks, repositories.Dpks)
737
- return dpks
738
-
739
- def get_triggers_repo(self, args):
740
- if args.project_name is not None:
741
- project = self.dl.projects.get(project_name=args.project_name)
742
- packages = project.packages
743
- services = project.services
744
- triggers = project.triggers
745
- else:
746
- try:
747
- packages = self.dl.projects.get().packages
748
- services = self.dl.projects.get().services
749
- triggers = self.dl.projects.get().triggers
750
- except Exception:
751
- packages = self.dl.packages
752
- services = self.dl.services
753
- triggers = self.dl.triggers
754
-
755
- if args.package_name is not None:
756
- package = packages.package_name(name=args.package_name)
757
- services = package.services
758
-
759
- if args.service_name is not None:
760
- service = services.get(service_name=args.service_name)
761
- triggers = service.triggers
762
-
763
- assert isinstance(triggers, repositories.Triggers)
764
- return triggers
1
+ import subprocess
2
+ import inquirer
3
+ import logging
4
+ import json
5
+ import os
6
+ import sys
7
+ import jwt
8
+
9
+ from .. import exceptions, entities, repositories, utilities, assets
10
+
11
+ logger = logging.getLogger(name='dtlpy')
12
+
13
+
14
+ class CommandExecutor:
15
+
16
+ def __init__(self, dl, parser):
17
+ self.dl = dl
18
+ self.parser = parser
19
+ self.utils = Utils(dl)
20
+
21
+ def run(self, args):
22
+ ########################
23
+ # Run Command if Exist #
24
+ ########################
25
+ if args.operation is None:
26
+ logger.info('See "dlp --help" for options')
27
+ return
28
+
29
+ operation = args.operation.lower().replace('-', '_')
30
+ if hasattr(self, operation):
31
+ getattr(self, operation)(args)
32
+ ###############
33
+ # Catch typos #
34
+ ###############
35
+ elif args.operation in ["project", 'dataset', 'item', 'service', 'package', 'video', 'deploy', 'generate']:
36
+ self.typos(args=args)
37
+ #######################
38
+ # Catch other options #
39
+ #######################
40
+ elif args.operation:
41
+ print('dlp: "%s" is not an dlp command' % args.operation)
42
+ print('See "dlp --help" for options')
43
+ else:
44
+ print('See "dlp --help" for options')
45
+
46
+ def help(self, args):
47
+ self.parser.print_help()
48
+
49
+ def logout(self, args):
50
+ self.dl.logout()
51
+ logger.info('logout successful')
52
+
53
+ # noinspection PyUnusedLocal
54
+ def login(self, args):
55
+ self.dl.login()
56
+ self.dl.info(with_token=False)
57
+
58
+ def login_token(self, args):
59
+ self.dl.login_token(args.token)
60
+ self.dl.info(with_token=False)
61
+
62
+ def login_api_key(self, args):
63
+ self.dl.login_api_key(api_key=args.api_key)
64
+
65
+ def login_secret(self, args):
66
+ self.login_m2m(args=args)
67
+
68
+ def login_m2m(self, args):
69
+ self.dl.login_m2m(email=args.email,
70
+ password=args.password,
71
+ client_id=args.client_id,
72
+ client_secret=args.client_secret)
73
+ self.dl.info(with_token=False)
74
+
75
+ def upgrade(self, args):
76
+ url = 'dtlpy'
77
+ if args.url is None:
78
+ try:
79
+ payload = jwt.decode(self.dl.client_api.token, algorithms=['HS256'],
80
+ verify=False, options={'verify_signature': False})
81
+ if 'admin' in payload['https://dataloop.ai/authorization']['roles']:
82
+ url = "https://storage.googleapis.com/dtlpy/dev/dtlpy-latest-py3-none-any.whl"
83
+ except Exception:
84
+ pass
85
+ else:
86
+ url = args.url
87
+
88
+ logger.info("Update DTLPy from {}".format(url))
89
+ logger.info("Installing using pip...")
90
+ cmd = "pip install {} --upgrade".format(url)
91
+ subprocess.Popen(cmd, shell=True)
92
+ sys.exit(0)
93
+
94
+ # noinspection PyUnusedLocal
95
+ def init(self, args):
96
+ self.dl.init()
97
+
98
+ # noinspection PyUnusedLocal
99
+ def checkout_state(self, args):
100
+ state = self.dl.checkout_state()
101
+ logger.info('Checked-out:')
102
+ for key, val in state.items():
103
+ try:
104
+ msg = '{entity} name: {name}\t{entity} id: {id}'.format(entity=key, name=val['name'], id=val['id'])
105
+ except KeyError:
106
+ msg = '{entity} Not Found'.format(entity=key)
107
+ logger.info(msg)
108
+
109
+ # noinspection PyUnusedLocal
110
+ def version(self, args):
111
+ logger.info("Dataloop SDK Version: {}".format(self.dl.__version__))
112
+
113
+ def development(self, args):
114
+ if args.development == "local":
115
+ # start the local development
116
+ if args.local == 'start':
117
+ development = self.utils.dl.client_api.state_io.get('development')
118
+ # create default values
119
+ if development is None:
120
+ development = dict()
121
+ if 'port' not in development:
122
+ development['port'] = 5802
123
+ if 'docker_image' not in development:
124
+ development['docker_image'] = 'dataloopai/dtlpy-agent:1.57.3.gpu.cuda11.5.py3.8.opencv'
125
+
126
+ # get values from input
127
+ if args.docker_image is not None:
128
+ development['docker_image'] = args.docker_image
129
+ if args.port is not None:
130
+ development['port'] = int(args.port)
131
+
132
+ # set values to local state
133
+ self.utils.dl.client_api.state_io.put('development', development)
134
+ utilities.local_development.start_session()
135
+ elif args.local == 'pause':
136
+ utilities.local_development.pause_session()
137
+ elif args.local == 'stop':
138
+ utilities.local_development.stop_session()
139
+ else:
140
+ print('Must select one of "start", "pause", "stop". Type "dlp development start --help" for options')
141
+ elif args.development == "remote":
142
+ logger.warning('FUTURE! This is not supported yet..')
143
+ else:
144
+ print('Type "dlp development --help" for options')
145
+
146
+ # noinspection PyUnusedLocal
147
+
148
+ def api(self, args):
149
+ if args.api == "info":
150
+ information = self.dl.info()
151
+ logger.info('-- Dataloop info --')
152
+ _ = [logger.info('{}: {}'.format(key, val)) for key, val in information.items()]
153
+
154
+ if args.api == "setenv":
155
+ self.dl.setenv(args.env)
156
+ logger.info("Platform environment: {}".format(self.dl.environment()))
157
+
158
+ def projects(self, args):
159
+ if args.projects == "ls":
160
+ self.dl.projects.list().print()
161
+ elif args.projects == "web":
162
+ if args.project_name is None:
163
+ args.project_name = self.dl.projects.get().name
164
+ self.dl.projects.open_in_web(project_name=args.project_name)
165
+ elif args.projects == "create":
166
+ project = self.dl.projects.create(args.project_name)
167
+ project.print()
168
+ project.checkout()
169
+ elif args.projects == "checkout":
170
+ self.dl.projects.checkout(project_name=args.project_name)
171
+ else:
172
+ print('Type "dlp projects --help" for options')
173
+
174
+ def datasets(self, args):
175
+ if args.datasets == "ls":
176
+ self.utils.get_datasets_repo(args=args).list().print()
177
+
178
+ elif args.datasets == "checkout":
179
+ self.dl.datasets.checkout(dataset_name=args.dataset_name)
180
+
181
+ elif args.datasets == "web":
182
+ if args.dataset_name is None:
183
+ args.dataset_name = self.dl.datasets.get().name
184
+ self.utils.get_datasets_repo(args=args).open_in_web(dataset_name=args.dataset_name)
185
+
186
+ elif args.datasets == "create":
187
+ dataset = self.utils.get_datasets_repo(args=args).create(dataset_name=args.dataset_name)
188
+ dataset.print()
189
+ if args.checkout:
190
+ dataset.checkout()
191
+ else:
192
+ print('Type "dlp datasets --help" for options')
193
+
194
+ def items(self, args):
195
+ if self.dl.token_expired():
196
+ logger.error("token expired, please login.")
197
+ return
198
+
199
+ if args.items == "ls":
200
+ project = self.dl.projects.get(project_name=args.project_name)
201
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
202
+ if isinstance(args.page, str):
203
+ try:
204
+ args.page = int(args.page)
205
+ except ValueError:
206
+ raise ValueError("Input --page must be integer")
207
+ filters = self.dl.Filters()
208
+
209
+ # add filters
210
+ if args.remote_path is not None:
211
+ if isinstance(args.remote_path, list):
212
+ filters.add(field="filename", values=args.remote_path, operator=entities.FiltersOperations.IN)
213
+ else:
214
+ filters.add(field="filename", values=args.remote_path)
215
+ if args.type is not None:
216
+ if isinstance(args.type, list):
217
+ filters.add(field='metadata.system.mimetype', values=args.type,
218
+ operator=entities.FiltersOperations.IN)
219
+ else:
220
+ filters.add(field="metadata.system.mimetype", values=args.type)
221
+
222
+ pages = dataset.items.list(filters=filters, page_offset=args.page)
223
+ pages.print()
224
+ print("Displaying page %d/%d" % (args.page, pages.total_pages_count - 1))
225
+
226
+ elif args.items == "web":
227
+ project = self.dl.projects.get(project_name=args.project_name)
228
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
229
+ dataset.items.open_in_web(filepath=args.remote_path)
230
+
231
+ elif args.items == "upload":
232
+ logger.info("Uploading directory...")
233
+ if isinstance(args.file_types, str):
234
+ args.file_types = [t.strip() for t in args.file_types.split(",")]
235
+ project = self.dl.projects.get(project_name=args.project_name)
236
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
237
+
238
+ dataset.items.upload(local_path=args.local_path,
239
+ remote_path=args.remote_path,
240
+ file_types=args.file_types,
241
+ overwrite=args.overwrite,
242
+ local_annotations_path=args.local_annotations_path)
243
+
244
+ elif args.items == "download":
245
+ logger.info("Downloading dataset...")
246
+ project = self.dl.projects.get(project_name=args.project_name)
247
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
248
+
249
+ annotation_options = None
250
+ if args.annotation_options is not None:
251
+ annotation_options = [t.strip() for t in args.annotation_options.split(",")]
252
+
253
+ annotation_filters = None
254
+ if args.annotation_filter_type is not None or args.annotation_filter_label is not None:
255
+ annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
256
+ if args.annotation_filter_type is not None:
257
+ annotation_filter_type = [t.strip() for t in args.annotation_filter_type.split(",")]
258
+ annotation_filters.add(field='type',
259
+ values=annotation_filter_type,
260
+ operator=entities.FiltersOperations.IN)
261
+ if args.annotation_filter_label is not None:
262
+ annotation_filter_label = [t.strip() for t in args.annotation_filter_label.split(",")]
263
+ annotation_filters.add(field='label',
264
+ values=annotation_filter_label,
265
+ operator=entities.FiltersOperations.IN)
266
+ # create remote path filters
267
+ filters = self.dl.Filters()
268
+ if args.remote_path is not None:
269
+ remote_path = [t.strip() for t in args.remote_path.split(",")]
270
+ if len(remote_path) == 1:
271
+ remote_path = remote_path[0]
272
+ filters.add(field="filename", values=remote_path)
273
+ elif len(remote_path) > 1:
274
+ for item in remote_path:
275
+ if '*' in item:
276
+ filters.add(field="dir", values=item, method='or')
277
+ remote_path.pop(remote_path.index(item))
278
+ filters.add(field="dir", values=remote_path, operator=entities.FiltersOperations.IN, method='or')
279
+
280
+ if not args.without_binaries:
281
+ dataset.items.download(filters=filters,
282
+ local_path=args.local_path,
283
+ annotation_options=annotation_options,
284
+ annotation_filters=annotation_filters,
285
+ overwrite=args.overwrite,
286
+ with_text=args.with_text,
287
+ thickness=int(args.thickness),
288
+ to_items_folder=not args.not_items_folder)
289
+ else:
290
+ dataset.download_annotations(filters=filters,
291
+ local_path=args.local_path,
292
+ annotation_options=annotation_options,
293
+ annotation_filters=annotation_filters,
294
+ overwrite=args.overwrite,
295
+ with_text=args.with_text,
296
+ thickness=int(args.thickness))
297
+
298
+ else:
299
+ print('Type "dlp items --help" for options')
300
+
301
+ def videos(self, args):
302
+ if self.dl.token_expired():
303
+ print("[ERROR] token expired, please login.")
304
+ return
305
+
306
+ if args.videos == "play":
307
+ from dtlpy.utilities.videos.video_player import VideoPlayer
308
+
309
+ VideoPlayer(
310
+ item_filepath=args.item_path,
311
+ dataset_name=args.dataset_name,
312
+ project_name=args.project_name,
313
+ )
314
+
315
+ elif args.videos == "upload":
316
+ if (
317
+ (args.split_chunks is not None)
318
+ or (args.split_seconds is not None)
319
+ or (args.split_times is not None)
320
+ ):
321
+ # upload with split
322
+ if isinstance(args.split_chunks, str):
323
+ args.split_chunks = int(args.split_chunks)
324
+ if isinstance(args.split_seconds, str):
325
+ args.split_seconds = int(args.split_seconds)
326
+ if isinstance(args.split_times, str):
327
+ args.split_times = [int(sec) for sec in args.split_times.split(",")]
328
+ self.dl.utilities.videos.Videos.split_and_upload(
329
+ project_name=args.project_name,
330
+ dataset_name=args.dataset_name,
331
+ filepath=args.filename,
332
+ remote_path=args.remote_path,
333
+ split_chunks=args.split_chunks,
334
+ split_seconds=args.split_seconds,
335
+ split_pairs=args.split_times,
336
+ )
337
+ else:
338
+ print('Type "dlp videos --help" for options')
339
+
340
+ def services(self, args):
341
+ if self.dl.token_expired():
342
+ print("[ERROR] token expired, please login.")
343
+ return
344
+
345
+ elif args.services == "delete":
346
+ service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
347
+ service.delete()
348
+ logger.info('Service: "{}" deleted successfully'.format(service.name))
349
+
350
+ elif args.services == "ls":
351
+ self.utils.get_services_repo(args=args).list().print()
352
+ elif args.services == "log":
353
+ project = self.dl.projects.get(project_name=args.project_name)
354
+ service = project.services.get(service_name=args.service_name)
355
+
356
+ logs = service.log(start=args.start)
357
+ try:
358
+ for log in logs:
359
+ if isinstance(log, list):
360
+ for log_record in log:
361
+ print(log_record)
362
+ else:
363
+ print(log)
364
+ except KeyboardInterrupt:
365
+ pass
366
+
367
+ elif args.services == "execute":
368
+ try:
369
+ if args.service_name:
370
+ service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
371
+ else:
372
+ service = self.dl.services.get()
373
+ except Exception:
374
+ logger.info('Please checkout a service')
375
+ return
376
+
377
+ if args.annotation_id is not None:
378
+ resource = 'Annotation'
379
+ elif args.item_id is not None:
380
+ resource = 'Item'
381
+ elif args.dataset_id is not None:
382
+ resource = 'Dataset'
383
+ else:
384
+ resource = None
385
+
386
+ try:
387
+ execution_input = json.loads(args.inputs)
388
+ except Exception:
389
+ logger.info('Input should be json serializable')
390
+ return
391
+ if len(execution_input) == 0:
392
+ execution_input = None
393
+
394
+ logger.info('executing')
395
+ service.execute(sync=not args.asynchronous,
396
+ execution_input=execution_input,
397
+ function_name=args.function_name,
398
+ resource=resource,
399
+ item_id=args.item_id,
400
+ dataset_id=args.dataset_id,
401
+ annotation_id=args.annotation_id)
402
+ logger.info("Successfully executed function: {}".format(args.function_name))
403
+
404
+ else:
405
+ logger.info('Type "dlp packages --help" for options')
406
+
407
+ def deploy(self, args):
408
+ project = self.dl.projects.get(project_name=args.project_name)
409
+ json_filepath = args.json_file
410
+ deployed_services, package = self.dl.packages.deploy_from_file(project=project, json_filepath=json_filepath)
411
+ logger.info("Successfully deployed {} from file: {}\nServices: {}".format(len(deployed_services),
412
+ json_filepath,
413
+ [s.name for s in deployed_services]))
414
+
415
+ def generate(self, args):
416
+ package_type = args.package_type if args.package_type else self.dl.PackageCatalog.DEFAULT_PACKAGE_TYPE
417
+ self.dl.packages.generate(name=args.package_name, src_path=os.getcwd(), package_type=package_type)
418
+ self.utils.dl.client_api.state_io.put('package', {'name': args.package_name})
419
+ logger.info('Successfully generated package files')
420
+
421
+ def triggers(self, args):
422
+
423
+ if args.triggers == "create":
424
+ args.actions = [t.strip() for t in args.actions.split(",")]
425
+ try:
426
+ service = self.utils.get_services_repo(args).get(service_name=args.service_name)
427
+ except exceptions.NotFound:
428
+ logger.critical('Service not found. Please check-out a service or provide valid service name')
429
+ return
430
+
431
+ trigger = service.triggers.create(name=args.name,
432
+ filters=json.loads('{}'.format(args.filters.replace("'", '"'))),
433
+ function_name=args.function_name,
434
+ resource=args.resource,
435
+ actions=args.actions)
436
+ logger.info('Trigger created successfully: {}'.format(trigger.name))
437
+
438
+ elif args.triggers == "delete":
439
+ triggers = self.utils.get_triggers_repo(args=args)
440
+ triggers.get(trigger_name=args.trigger_name).delete()
441
+ logger.info('Trigger deleted successfully: {}'.format(args.trigger_name))
442
+ elif args.triggers == "ls":
443
+ triggers = self.utils.get_triggers_repo(args=args)
444
+ triggers.list().print()
445
+ else:
446
+ logger.info('Type "dlp packages --help" for options')
447
+
448
+ def packages(self, args):
449
+ if self.dl.token_expired():
450
+ logger.error("token expired, please login.")
451
+ return
452
+
453
+ elif args.packages == "delete":
454
+ package = self.utils.get_packages_repo(args=args).get(package_name=args.package_name)
455
+ package.delete()
456
+ logger.info('Successfully deleted package {}'.format(package.name))
457
+
458
+ elif args.packages == "ls":
459
+ self.utils.get_packages_repo(args=args).list().print()
460
+
461
+ elif args.packages == "checkout":
462
+ self.dl.packages.checkout(package_name=args.package_name)
463
+
464
+ elif args.packages == "push":
465
+ packages = self.utils.get_packages_repo(args=args)
466
+
467
+ package = packages.push(src_path=args.src_path,
468
+ package_name=args.package_name,
469
+ checkout=args.checkout)
470
+
471
+ logger.info("Successfully pushed package to platform\n"
472
+ "Package id:{}\nPackage version:{}".format(package.id,
473
+ package.version))
474
+ elif args.packages == "deploy":
475
+ packages = self.utils.get_packages_repo(args=args)
476
+
477
+ package = packages.deploy(package_name=args.package_name,
478
+ checkout=args.checkout,
479
+ module_name=args.module_name)
480
+
481
+ logger.info("Successfully pushed package to platform\n"
482
+ "Package id:{}\nPackage version:{}".format(package.id,
483
+ package.version))
484
+
485
+ elif args.packages == "test":
486
+ go_back = False
487
+ if 'src' in os.listdir(os.getcwd()):
488
+ go_back = True
489
+ os.chdir('./src')
490
+ try:
491
+ self.dl.packages.test_local_package(concurrency=int(args.concurrency),
492
+ function_name=args.function_name)
493
+ except Exception:
494
+ logger.exception('failed during test')
495
+ finally:
496
+ if go_back:
497
+ os.chdir('..')
498
+
499
+ else:
500
+ logger.info('Type "dlp packages --help" for options')
501
+
502
+ def app(self, args):
503
+ if args.app == 'pack':
504
+ path = self.dl.dpks.pack()
505
+ logger.info(f'Packed to {path}')
506
+ elif args.app == 'init':
507
+ app_filename = assets.paths.APP_JSON_FILENAME
508
+ if os.path.isfile(app_filename):
509
+ questions = [
510
+ inquirer.Confirm(name='overwrite',
511
+ message=f"Dataloop app already initialized. Re-initialize?",
512
+ default=False)]
513
+ answers = inquirer.prompt(questions)
514
+ if answers.get('overwrite') is False:
515
+ return
516
+ name = args.name
517
+ description = args.description
518
+ attributes = args.attributes
519
+ icon = args.icon
520
+ scope = args.scope
521
+ as_array = [name, description, attributes, icon, scope]
522
+ if as_array.count(None) == len(as_array): # No one value is initialized
523
+ dir_name = os.path.basename(os.getcwd())
524
+ questions = [
525
+ inquirer.Text(name='name',
526
+ message=f"Enter the name of the app (or press enter for '{dir_name}'):",
527
+ default=dir_name),
528
+ inquirer.Text(name='description',
529
+ message="Enter the description (or enter for empty): "),
530
+ inquirer.Text(name='attributes',
531
+ message="Enter the attributes (an object, or enter for empty): ",
532
+ default=None),
533
+ inquirer.Text(name='icon',
534
+ message="Enter the path to the icon (or enter for empty): "),
535
+ inquirer.Text(name='scope',
536
+ message="Enter the scope (or enter for 'organization'): ",
537
+ default='project'),
538
+ ]
539
+ answers = inquirer.prompt(questions)
540
+ name = answers.get('name')
541
+ description = answers.get('description')
542
+ attributes = answers.get('attributes')
543
+ icon = answers.get('icon')
544
+ scope = answers.get('scope')
545
+
546
+ if attributes is not None:
547
+ attributes = json.loads(attributes)
548
+
549
+ self.dl.dpks.init(name=name,
550
+ description=description,
551
+ attributes=attributes,
552
+ icon=icon,
553
+ scope=scope)
554
+ elif args.app == 'add':
555
+ if args.panel is True:
556
+ default_panel_name = "myPanel"
557
+ choices = list(entities.dpk.SlotType)
558
+
559
+ questions = [
560
+ inquirer.Text(name='name',
561
+ message=f"Enter PANEL NAME (or press enter for '{default_panel_name}'): ",
562
+ default='default_panel_name'),
563
+ inquirer.List(name='support_slot_type',
564
+ message="Enter SUPPORTED SLOT TYPE:",
565
+ choices=choices),
566
+ ]
567
+
568
+ answers = inquirer.prompt(questions)
569
+ #####
570
+ # create a dir for that panel
571
+ os.makedirs(answers.get('name'), exist_ok=True)
572
+ # dump to dataloop.json
573
+ app_filename = assets.paths.APP_JSON_FILENAME
574
+ if not os.path.isfile(app_filename):
575
+ logger.error(f"Can't find app config file ({app_filename}), please run `dlp app init` first")
576
+ else:
577
+ with open(app_filename, 'r') as f:
578
+ dpk = entities.Dpk.from_json(json.load(f))
579
+ dpk.components.panels.append(entities.Panel(name=answers.get('name'),
580
+ supported_slots=[answers.get('support_slot_type')]))
581
+ with open(app_filename, 'w') as f:
582
+ json.dump(dpk.to_json(), f, indent=4)
583
+
584
+ elif args.app == 'publish':
585
+ dpk = self.utils.get_dpks_repo(args).publish()
586
+ if dpk:
587
+ logger.info(f'Published the application: id={dpk.id}')
588
+ else:
589
+ logger.info("Couldn't publish the application")
590
+ elif args.app == 'update':
591
+ # TODO: I think it changed, not implemented
592
+ logger.info('App updated successfully')
593
+ elif args.app == 'resume':
594
+ succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
595
+ if succeed is True:
596
+ logger.info('Resumed application successfully')
597
+ else:
598
+ logger.info('Application resume failed')
599
+ elif args.app == 'pause':
600
+ succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
601
+ if succeed is True:
602
+ logger.info('Paused application successfully')
603
+ else:
604
+ logger.info('Application pause failed')
605
+ elif args.app == 'install':
606
+ app = self.utils.get_apps_repo(args).install(
607
+ dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
608
+ organization_id=args.org_id
609
+ )
610
+ if app is not None:
611
+ logger.info('App installed successfully')
612
+ else:
613
+ logger.info("Couldn't install the app")
614
+ elif args.app == 'pull':
615
+ succeed = self.dl.dpks.pull(dpk_name=args.dpk_name)
616
+ if succeed is True:
617
+ logger.info("Pulled successfully")
618
+ else:
619
+ logger.info("Couldn't pull")
620
+ elif args.app == 'list':
621
+ self.dl.apps.list().print()
622
+
623
+ # noinspection PyUnusedLocal
624
+ @staticmethod
625
+ def pwd(args):
626
+ print(os.getcwd())
627
+
628
+ @staticmethod
629
+ def cd(args):
630
+ directory = args.dir
631
+ if directory == '..':
632
+ directory = os.path.split(os.getcwd())[0]
633
+ os.chdir(directory)
634
+ print(os.getcwd())
635
+
636
+ @staticmethod
637
+ def mkdir(args):
638
+ os.mkdir(args.name)
639
+
640
+ # noinspection PyUnusedLocal
641
+ @staticmethod
642
+ def ls(args):
643
+ print(os.getcwd())
644
+ dirs = os.listdir(os.getcwd())
645
+ import pprint
646
+
647
+ pp = pprint.PrettyPrinter(indent=3)
648
+ pp.pprint(dirs)
649
+
650
+ # noinspection PyUnusedLocal
651
+ @staticmethod
652
+ def clear(args):
653
+ if os.name == "nt":
654
+ os.system('cls')
655
+ else:
656
+ os.system('clear')
657
+
658
+ @staticmethod
659
+ def typos(args):
660
+ print('dlp: "{op}" is not an dlp command. Did you mean "{op}s"?'.format(op=args.operation))
661
+
662
+
663
+ class Utils:
664
+ def __init__(self, dl):
665
+ self.dl = dl
666
+
667
+ def get_packages_repo(self, args):
668
+ if args.project_name is not None:
669
+ project = self.dl.projects.get(project_name=args.project_name)
670
+ packages = project.packages
671
+ else:
672
+ try:
673
+ packages = self.dl.projects.get().packages
674
+ except Exception:
675
+ packages = self.dl.packages
676
+
677
+ assert isinstance(packages, repositories.Packages)
678
+ return packages
679
+
680
+ def get_datasets_repo(self, args):
681
+ if args.project_name is not None:
682
+ project = self.dl.projects.get(project_name=args.project_name)
683
+ datasets = project.datasets
684
+ else:
685
+ try:
686
+ datasets = self.dl.projects.get().datasets
687
+ except Exception:
688
+ datasets = self.dl.datasets
689
+
690
+ assert isinstance(datasets, repositories.Datasets)
691
+ return datasets
692
+
693
+ def get_services_repo(self, args):
694
+ if args.project_name is not None:
695
+ project = self.dl.projects.get(project_name=args.project_name)
696
+ packages = project.packages
697
+ services = project.services
698
+ else:
699
+ try:
700
+ packages = self.dl.projects.get().packages
701
+ services = self.dl.projects.get().services
702
+ except Exception:
703
+ packages = self.dl.packages
704
+ services = self.dl.services
705
+
706
+ if args.package_name is not None:
707
+ package = packages.get(package_name=args.package_name)
708
+ services = package.services
709
+
710
+ assert isinstance(services, repositories.Services)
711
+ return services
712
+
713
+ def get_apps_repo(self, args) -> repositories.Apps:
714
+ if args.project_name is not None:
715
+ project = self.dl.projects.get(project_name=args.project_name)
716
+ apps = project.apps
717
+ else:
718
+ try:
719
+ apps = self.dl.projects.get().apps
720
+ except Exception:
721
+ apps = self.dl.apps
722
+ assert isinstance(apps, repositories.Apps)
723
+ return apps
724
+
725
+ def get_dpks_repo(self, args) -> repositories.Dpks:
726
+ if args.project_name is None and args.project_id is None:
727
+ try:
728
+ dpks = self.dl.projects.get().dpks
729
+ except Exception:
730
+ dpks = self.dl.dpks
731
+ else:
732
+ project = self.dl.projects.get(project_name=args.project_name, project_id=args.project_id)
733
+ dpks = project.dpks
734
+ if dpks.project is None:
735
+ raise ValueError("Must input one of `project-name` or `project-id`") from None
736
+ assert isinstance(dpks, repositories.Dpks)
737
+ return dpks
738
+
739
+ def get_triggers_repo(self, args):
740
+ if args.project_name is not None:
741
+ project = self.dl.projects.get(project_name=args.project_name)
742
+ packages = project.packages
743
+ services = project.services
744
+ triggers = project.triggers
745
+ else:
746
+ try:
747
+ packages = self.dl.projects.get().packages
748
+ services = self.dl.projects.get().services
749
+ triggers = self.dl.projects.get().triggers
750
+ except Exception:
751
+ packages = self.dl.packages
752
+ services = self.dl.services
753
+ triggers = self.dl.triggers
754
+
755
+ if args.package_name is not None:
756
+ package = packages.package_name(name=args.package_name)
757
+ services = package.services
758
+
759
+ if args.service_name is not None:
760
+ service = services.get(service_name=args.service_name)
761
+ triggers = service.triggers
762
+
763
+ assert isinstance(triggers, repositories.Triggers)
764
+ return triggers