dtlpy 1.115.44__py3-none-any.whl → 1.116.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +145 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +959 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +963 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1257 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1504 -1504
  167. dtlpy/repositories/downloader.py +976 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +226 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +419 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1785 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.116.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
@@ -1,764 +1,764 @@
1
- import subprocess
2
- import inquirer
3
- import logging
4
- import json
5
- import os
6
- import sys
7
- import jwt
8
-
9
- from .. import exceptions, entities, repositories, utilities, assets
10
-
11
- logger = logging.getLogger(name='dtlpy')
12
-
13
-
14
- class CommandExecutor:
15
-
16
- def __init__(self, dl, parser):
17
- self.dl = dl
18
- self.parser = parser
19
- self.utils = Utils(dl)
20
-
21
- def run(self, args):
22
- ########################
23
- # Run Command if Exist #
24
- ########################
25
- if args.operation is None:
26
- logger.info('See "dlp --help" for options')
27
- return
28
-
29
- operation = args.operation.lower().replace('-', '_')
30
- if hasattr(self, operation):
31
- getattr(self, operation)(args)
32
- ###############
33
- # Catch typos #
34
- ###############
35
- elif args.operation in ["project", 'dataset', 'item', 'service', 'package', 'video', 'deploy', 'generate']:
36
- self.typos(args=args)
37
- #######################
38
- # Catch other options #
39
- #######################
40
- elif args.operation:
41
- print('dlp: "%s" is not an dlp command' % args.operation)
42
- print('See "dlp --help" for options')
43
- else:
44
- print('See "dlp --help" for options')
45
-
46
- def help(self, args):
47
- self.parser.print_help()
48
-
49
- def logout(self, args):
50
- self.dl.logout()
51
- logger.info('logout successful')
52
-
53
- # noinspection PyUnusedLocal
54
- def login(self, args):
55
- self.dl.login()
56
- self.dl.info(with_token=False)
57
-
58
- def login_token(self, args):
59
- self.dl.login_token(args.token)
60
- self.dl.info(with_token=False)
61
-
62
- def login_api_key(self, args):
63
- self.dl.login_api_key(api_key=args.api_key)
64
-
65
- def login_secret(self, args):
66
- self.login_m2m(args=args)
67
-
68
- def login_m2m(self, args):
69
- self.dl.login_m2m(email=args.email,
70
- password=args.password,
71
- client_id=args.client_id,
72
- client_secret=args.client_secret)
73
- self.dl.info(with_token=False)
74
-
75
- def upgrade(self, args):
76
- url = 'dtlpy'
77
- if args.url is None:
78
- try:
79
- payload = jwt.decode(self.dl.client_api.token, algorithms=['HS256'],
80
- verify=False, options={'verify_signature': False})
81
- if 'admin' in payload['https://dataloop.ai/authorization']['roles']:
82
- url = "https://storage.googleapis.com/dtlpy/dev/dtlpy-latest-py3-none-any.whl"
83
- except Exception:
84
- pass
85
- else:
86
- url = args.url
87
-
88
- logger.info("Update DTLPy from {}".format(url))
89
- logger.info("Installing using pip...")
90
- cmd = "pip install {} --upgrade".format(url)
91
- subprocess.Popen(cmd, shell=True)
92
- sys.exit(0)
93
-
94
- # noinspection PyUnusedLocal
95
- def init(self, args):
96
- self.dl.init()
97
-
98
- # noinspection PyUnusedLocal
99
- def checkout_state(self, args):
100
- state = self.dl.checkout_state()
101
- logger.info('Checked-out:')
102
- for key, val in state.items():
103
- try:
104
- msg = '{entity} name: {name}\t{entity} id: {id}'.format(entity=key, name=val['name'], id=val['id'])
105
- except KeyError:
106
- msg = '{entity} Not Found'.format(entity=key)
107
- logger.info(msg)
108
-
109
- # noinspection PyUnusedLocal
110
- def version(self, args):
111
- logger.info("Dataloop SDK Version: {}".format(self.dl.__version__))
112
-
113
- def development(self, args):
114
- if args.development == "local":
115
- # start the local development
116
- if args.local == 'start':
117
- development = self.utils.dl.client_api.state_io.get('development')
118
- # create default values
119
- if development is None:
120
- development = dict()
121
- if 'port' not in development:
122
- development['port'] = 5802
123
- if 'docker_image' not in development:
124
- development['docker_image'] = 'dataloopai/dtlpy-agent:1.57.3.gpu.cuda11.5.py3.8.opencv'
125
-
126
- # get values from input
127
- if args.docker_image is not None:
128
- development['docker_image'] = args.docker_image
129
- if args.port is not None:
130
- development['port'] = int(args.port)
131
-
132
- # set values to local state
133
- self.utils.dl.client_api.state_io.put('development', development)
134
- utilities.local_development.start_session()
135
- elif args.local == 'pause':
136
- utilities.local_development.pause_session()
137
- elif args.local == 'stop':
138
- utilities.local_development.stop_session()
139
- else:
140
- print('Must select one of "start", "pause", "stop". Type "dlp development start --help" for options')
141
- elif args.development == "remote":
142
- logger.warning('FUTURE! This is not supported yet..')
143
- else:
144
- print('Type "dlp development --help" for options')
145
-
146
- # noinspection PyUnusedLocal
147
-
148
- def api(self, args):
149
- if args.api == "info":
150
- information = self.dl.info()
151
- logger.info('-- Dataloop info --')
152
- _ = [logger.info('{}: {}'.format(key, val)) for key, val in information.items()]
153
-
154
- if args.api == "setenv":
155
- self.dl.setenv(args.env)
156
- logger.info("Platform environment: {}".format(self.dl.environment()))
157
-
158
- def projects(self, args):
159
- if args.projects == "ls":
160
- self.dl.projects.list().print()
161
- elif args.projects == "web":
162
- if args.project_name is None:
163
- args.project_name = self.dl.projects.get().name
164
- self.dl.projects.open_in_web(project_name=args.project_name)
165
- elif args.projects == "create":
166
- project = self.dl.projects.create(args.project_name)
167
- project.print()
168
- project.checkout()
169
- elif args.projects == "checkout":
170
- self.dl.projects.checkout(project_name=args.project_name)
171
- else:
172
- print('Type "dlp projects --help" for options')
173
-
174
- def datasets(self, args):
175
- if args.datasets == "ls":
176
- self.utils.get_datasets_repo(args=args).list().print()
177
-
178
- elif args.datasets == "checkout":
179
- self.dl.datasets.checkout(dataset_name=args.dataset_name)
180
-
181
- elif args.datasets == "web":
182
- if args.dataset_name is None:
183
- args.dataset_name = self.dl.datasets.get().name
184
- self.utils.get_datasets_repo(args=args).open_in_web(dataset_name=args.dataset_name)
185
-
186
- elif args.datasets == "create":
187
- dataset = self.utils.get_datasets_repo(args=args).create(dataset_name=args.dataset_name)
188
- dataset.print()
189
- if args.checkout:
190
- dataset.checkout()
191
- else:
192
- print('Type "dlp datasets --help" for options')
193
-
194
- def items(self, args):
195
- if self.dl.token_expired():
196
- logger.error("token expired, please login.")
197
- return
198
-
199
- if args.items == "ls":
200
- project = self.dl.projects.get(project_name=args.project_name)
201
- dataset = project.datasets.get(dataset_name=args.dataset_name)
202
- if isinstance(args.page, str):
203
- try:
204
- args.page = int(args.page)
205
- except ValueError:
206
- raise ValueError("Input --page must be integer")
207
- filters = self.dl.Filters()
208
-
209
- # add filters
210
- if args.remote_path is not None:
211
- if isinstance(args.remote_path, list):
212
- filters.add(field="filename", values=args.remote_path, operator=entities.FiltersOperations.IN)
213
- else:
214
- filters.add(field="filename", values=args.remote_path)
215
- if args.type is not None:
216
- if isinstance(args.type, list):
217
- filters.add(field='metadata.system.mimetype', values=args.type,
218
- operator=entities.FiltersOperations.IN)
219
- else:
220
- filters.add(field="metadata.system.mimetype", values=args.type)
221
-
222
- pages = dataset.items.list(filters=filters, page_offset=args.page)
223
- pages.print()
224
- print("Displaying page %d/%d" % (args.page, pages.total_pages_count - 1))
225
-
226
- elif args.items == "web":
227
- project = self.dl.projects.get(project_name=args.project_name)
228
- dataset = project.datasets.get(dataset_name=args.dataset_name)
229
- dataset.items.open_in_web(filepath=args.remote_path)
230
-
231
- elif args.items == "upload":
232
- logger.info("Uploading directory...")
233
- if isinstance(args.file_types, str):
234
- args.file_types = [t.strip() for t in args.file_types.split(",")]
235
- project = self.dl.projects.get(project_name=args.project_name)
236
- dataset = project.datasets.get(dataset_name=args.dataset_name)
237
-
238
- dataset.items.upload(local_path=args.local_path,
239
- remote_path=args.remote_path,
240
- file_types=args.file_types,
241
- overwrite=args.overwrite,
242
- local_annotations_path=args.local_annotations_path)
243
-
244
- elif args.items == "download":
245
- logger.info("Downloading dataset...")
246
- project = self.dl.projects.get(project_name=args.project_name)
247
- dataset = project.datasets.get(dataset_name=args.dataset_name)
248
-
249
- annotation_options = None
250
- if args.annotation_options is not None:
251
- annotation_options = [t.strip() for t in args.annotation_options.split(",")]
252
-
253
- annotation_filters = None
254
- if args.annotation_filter_type is not None or args.annotation_filter_label is not None:
255
- annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
256
- if args.annotation_filter_type is not None:
257
- annotation_filter_type = [t.strip() for t in args.annotation_filter_type.split(",")]
258
- annotation_filters.add(field='type',
259
- values=annotation_filter_type,
260
- operator=entities.FiltersOperations.IN)
261
- if args.annotation_filter_label is not None:
262
- annotation_filter_label = [t.strip() for t in args.annotation_filter_label.split(",")]
263
- annotation_filters.add(field='label',
264
- values=annotation_filter_label,
265
- operator=entities.FiltersOperations.IN)
266
- # create remote path filters
267
- filters = self.dl.Filters()
268
- if args.remote_path is not None:
269
- remote_path = [t.strip() for t in args.remote_path.split(",")]
270
- if len(remote_path) == 1:
271
- remote_path = remote_path[0]
272
- filters.add(field="filename", values=remote_path)
273
- elif len(remote_path) > 1:
274
- for item in remote_path:
275
- if '*' in item:
276
- filters.add(field="dir", values=item, method='or')
277
- remote_path.pop(remote_path.index(item))
278
- filters.add(field="dir", values=remote_path, operator=entities.FiltersOperations.IN, method='or')
279
-
280
- if not args.without_binaries:
281
- dataset.items.download(filters=filters,
282
- local_path=args.local_path,
283
- annotation_options=annotation_options,
284
- annotation_filters=annotation_filters,
285
- overwrite=args.overwrite,
286
- with_text=args.with_text,
287
- thickness=int(args.thickness),
288
- to_items_folder=not args.not_items_folder)
289
- else:
290
- dataset.download_annotations(filters=filters,
291
- local_path=args.local_path,
292
- annotation_options=annotation_options,
293
- annotation_filters=annotation_filters,
294
- overwrite=args.overwrite,
295
- with_text=args.with_text,
296
- thickness=int(args.thickness))
297
-
298
- else:
299
- print('Type "dlp items --help" for options')
300
-
301
- def videos(self, args):
302
- if self.dl.token_expired():
303
- print("[ERROR] token expired, please login.")
304
- return
305
-
306
- if args.videos == "play":
307
- from dtlpy.utilities.videos.video_player import VideoPlayer
308
-
309
- VideoPlayer(
310
- item_filepath=args.item_path,
311
- dataset_name=args.dataset_name,
312
- project_name=args.project_name,
313
- )
314
-
315
- elif args.videos == "upload":
316
- if (
317
- (args.split_chunks is not None)
318
- or (args.split_seconds is not None)
319
- or (args.split_times is not None)
320
- ):
321
- # upload with split
322
- if isinstance(args.split_chunks, str):
323
- args.split_chunks = int(args.split_chunks)
324
- if isinstance(args.split_seconds, str):
325
- args.split_seconds = int(args.split_seconds)
326
- if isinstance(args.split_times, str):
327
- args.split_times = [int(sec) for sec in args.split_times.split(",")]
328
- self.dl.utilities.videos.Videos.split_and_upload(
329
- project_name=args.project_name,
330
- dataset_name=args.dataset_name,
331
- filepath=args.filename,
332
- remote_path=args.remote_path,
333
- split_chunks=args.split_chunks,
334
- split_seconds=args.split_seconds,
335
- split_pairs=args.split_times,
336
- )
337
- else:
338
- print('Type "dlp videos --help" for options')
339
-
340
- def services(self, args):
341
- if self.dl.token_expired():
342
- print("[ERROR] token expired, please login.")
343
- return
344
-
345
- elif args.services == "delete":
346
- service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
347
- service.delete()
348
- logger.info('Service: "{}" deleted successfully'.format(service.name))
349
-
350
- elif args.services == "ls":
351
- self.utils.get_services_repo(args=args).list().print()
352
- elif args.services == "log":
353
- project = self.dl.projects.get(project_name=args.project_name)
354
- service = project.services.get(service_name=args.service_name)
355
-
356
- logs = service.log(start=args.start)
357
- try:
358
- for log in logs:
359
- if isinstance(log, list):
360
- for log_record in log:
361
- print(log_record)
362
- else:
363
- print(log)
364
- except KeyboardInterrupt:
365
- pass
366
-
367
- elif args.services == "execute":
368
- try:
369
- if args.service_name:
370
- service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
371
- else:
372
- service = self.dl.services.get()
373
- except Exception:
374
- logger.info('Please checkout a service')
375
- return
376
-
377
- if args.annotation_id is not None:
378
- resource = 'Annotation'
379
- elif args.item_id is not None:
380
- resource = 'Item'
381
- elif args.dataset_id is not None:
382
- resource = 'Dataset'
383
- else:
384
- resource = None
385
-
386
- try:
387
- execution_input = json.loads(args.inputs)
388
- except Exception:
389
- logger.info('Input should be json serializable')
390
- return
391
- if len(execution_input) == 0:
392
- execution_input = None
393
-
394
- logger.info('executing')
395
- service.execute(sync=not args.asynchronous,
396
- execution_input=execution_input,
397
- function_name=args.function_name,
398
- resource=resource,
399
- item_id=args.item_id,
400
- dataset_id=args.dataset_id,
401
- annotation_id=args.annotation_id)
402
- logger.info("Successfully executed function: {}".format(args.function_name))
403
-
404
- else:
405
- logger.info('Type "dlp packages --help" for options')
406
-
407
- def deploy(self, args):
408
- project = self.dl.projects.get(project_name=args.project_name)
409
- json_filepath = args.json_file
410
- deployed_services, package = self.dl.packages.deploy_from_file(project=project, json_filepath=json_filepath)
411
- logger.info("Successfully deployed {} from file: {}\nServices: {}".format(len(deployed_services),
412
- json_filepath,
413
- [s.name for s in deployed_services]))
414
-
415
- def generate(self, args):
416
- package_type = args.package_type if args.package_type else self.dl.PackageCatalog.DEFAULT_PACKAGE_TYPE
417
- self.dl.packages.generate(name=args.package_name, src_path=os.getcwd(), package_type=package_type)
418
- self.utils.dl.client_api.state_io.put('package', {'name': args.package_name})
419
- logger.info('Successfully generated package files')
420
-
421
- def triggers(self, args):
422
-
423
- if args.triggers == "create":
424
- args.actions = [t.strip() for t in args.actions.split(",")]
425
- try:
426
- service = self.utils.get_services_repo(args).get(service_name=args.service_name)
427
- except exceptions.NotFound:
428
- logger.critical('Service not found. Please check-out a service or provide valid service name')
429
- return
430
-
431
- trigger = service.triggers.create(name=args.name,
432
- filters=json.loads('{}'.format(args.filters.replace("'", '"'))),
433
- function_name=args.function_name,
434
- resource=args.resource,
435
- actions=args.actions)
436
- logger.info('Trigger created successfully: {}'.format(trigger.name))
437
-
438
- elif args.triggers == "delete":
439
- triggers = self.utils.get_triggers_repo(args=args)
440
- triggers.get(trigger_name=args.trigger_name).delete()
441
- logger.info('Trigger deleted successfully: {}'.format(args.trigger_name))
442
- elif args.triggers == "ls":
443
- triggers = self.utils.get_triggers_repo(args=args)
444
- triggers.list().print()
445
- else:
446
- logger.info('Type "dlp packages --help" for options')
447
-
448
- def packages(self, args):
449
- if self.dl.token_expired():
450
- logger.error("token expired, please login.")
451
- return
452
-
453
- elif args.packages == "delete":
454
- package = self.utils.get_packages_repo(args=args).get(package_name=args.package_name)
455
- package.delete()
456
- logger.info('Successfully deleted package {}'.format(package.name))
457
-
458
- elif args.packages == "ls":
459
- self.utils.get_packages_repo(args=args).list().print()
460
-
461
- elif args.packages == "checkout":
462
- self.dl.packages.checkout(package_name=args.package_name)
463
-
464
- elif args.packages == "push":
465
- packages = self.utils.get_packages_repo(args=args)
466
-
467
- package = packages.push(src_path=args.src_path,
468
- package_name=args.package_name,
469
- checkout=args.checkout)
470
-
471
- logger.info("Successfully pushed package to platform\n"
472
- "Package id:{}\nPackage version:{}".format(package.id,
473
- package.version))
474
- elif args.packages == "deploy":
475
- packages = self.utils.get_packages_repo(args=args)
476
-
477
- package = packages.deploy(package_name=args.package_name,
478
- checkout=args.checkout,
479
- module_name=args.module_name)
480
-
481
- logger.info("Successfully pushed package to platform\n"
482
- "Package id:{}\nPackage version:{}".format(package.id,
483
- package.version))
484
-
485
- elif args.packages == "test":
486
- go_back = False
487
- if 'src' in os.listdir(os.getcwd()):
488
- go_back = True
489
- os.chdir('./src')
490
- try:
491
- self.dl.packages.test_local_package(concurrency=int(args.concurrency),
492
- function_name=args.function_name)
493
- except Exception:
494
- logger.exception('failed during test')
495
- finally:
496
- if go_back:
497
- os.chdir('..')
498
-
499
- else:
500
- logger.info('Type "dlp packages --help" for options')
501
-
502
- def app(self, args):
503
- if args.app == 'pack':
504
- path = self.dl.dpks.pack()
505
- logger.info(f'Packed to {path}')
506
- elif args.app == 'init':
507
- app_filename = assets.paths.APP_JSON_FILENAME
508
- if os.path.isfile(app_filename):
509
- questions = [
510
- inquirer.Confirm(name='overwrite',
511
- message=f"Dataloop app already initialized. Re-initialize?",
512
- default=False)]
513
- answers = inquirer.prompt(questions)
514
- if answers.get('overwrite') is False:
515
- return
516
- name = args.name
517
- description = args.description
518
- attributes = args.attributes
519
- icon = args.icon
520
- scope = args.scope
521
- as_array = [name, description, attributes, icon, scope]
522
- if as_array.count(None) == len(as_array): # No one value is initialized
523
- dir_name = os.path.basename(os.getcwd())
524
- questions = [
525
- inquirer.Text(name='name',
526
- message=f"Enter the name of the app (or press enter for '{dir_name}'):",
527
- default=dir_name),
528
- inquirer.Text(name='description',
529
- message="Enter the description (or enter for empty): "),
530
- inquirer.Text(name='attributes',
531
- message="Enter the attributes (an object, or enter for empty): ",
532
- default=None),
533
- inquirer.Text(name='icon',
534
- message="Enter the path to the icon (or enter for empty): "),
535
- inquirer.Text(name='scope',
536
- message="Enter the scope (or enter for 'organization'): ",
537
- default='project'),
538
- ]
539
- answers = inquirer.prompt(questions)
540
- name = answers.get('name')
541
- description = answers.get('description')
542
- attributes = answers.get('attributes')
543
- icon = answers.get('icon')
544
- scope = answers.get('scope')
545
-
546
- if attributes is not None:
547
- attributes = json.loads(attributes)
548
-
549
- self.dl.dpks.init(name=name,
550
- description=description,
551
- attributes=attributes,
552
- icon=icon,
553
- scope=scope)
554
- elif args.app == 'add':
555
- if args.panel is True:
556
- default_panel_name = "myPanel"
557
- choices = list(entities.dpk.SlotType)
558
-
559
- questions = [
560
- inquirer.Text(name='name',
561
- message=f"Enter PANEL NAME (or press enter for '{default_panel_name}'): ",
562
- default='default_panel_name'),
563
- inquirer.List(name='support_slot_type',
564
- message="Enter SUPPORTED SLOT TYPE:",
565
- choices=choices),
566
- ]
567
-
568
- answers = inquirer.prompt(questions)
569
- #####
570
- # create a dir for that panel
571
- os.makedirs(answers.get('name'), exist_ok=True)
572
- # dump to dataloop.json
573
- app_filename = assets.paths.APP_JSON_FILENAME
574
- if not os.path.isfile(app_filename):
575
- logger.error(f"Can't find app config file ({app_filename}), please run `dlp app init` first")
576
- else:
577
- with open(app_filename, 'r') as f:
578
- dpk = entities.Dpk.from_json(json.load(f))
579
- dpk.components.panels.append(entities.Panel(name=answers.get('name'),
580
- supported_slots=[answers.get('support_slot_type')]))
581
- with open(app_filename, 'w') as f:
582
- json.dump(dpk.to_json(), f, indent=4)
583
-
584
- elif args.app == 'publish':
585
- dpk = self.utils.get_dpks_repo(args).publish()
586
- if dpk:
587
- logger.info(f'Published the application: id={dpk.id}')
588
- else:
589
- logger.info("Couldn't publish the application")
590
- elif args.app == 'update':
591
- # TODO: I think it changed, not implemented
592
- logger.info('App updated successfully')
593
- elif args.app == 'resume':
594
- succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
595
- if succeed is True:
596
- logger.info('Resumed application successfully')
597
- else:
598
- logger.info('Application resume failed')
599
- elif args.app == 'pause':
600
- succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
601
- if succeed is True:
602
- logger.info('Paused application successfully')
603
- else:
604
- logger.info('Application pause failed')
605
- elif args.app == 'install':
606
- app = self.utils.get_apps_repo(args).install(
607
- dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
608
- organization_id=args.org_id
609
- )
610
- if app is not None:
611
- logger.info('App installed successfully')
612
- else:
613
- logger.info("Couldn't install the app")
614
- elif args.app == 'pull':
615
- succeed = self.dl.dpks.pull(dpk_name=args.dpk_name)
616
- if succeed is True:
617
- logger.info("Pulled successfully")
618
- else:
619
- logger.info("Couldn't pull")
620
- elif args.app == 'list':
621
- self.dl.apps.list().print()
622
-
623
- # noinspection PyUnusedLocal
624
- @staticmethod
625
- def pwd(args):
626
- print(os.getcwd())
627
-
628
- @staticmethod
629
- def cd(args):
630
- directory = args.dir
631
- if directory == '..':
632
- directory = os.path.split(os.getcwd())[0]
633
- os.chdir(directory)
634
- print(os.getcwd())
635
-
636
- @staticmethod
637
- def mkdir(args):
638
- os.mkdir(args.name)
639
-
640
- # noinspection PyUnusedLocal
641
- @staticmethod
642
- def ls(args):
643
- print(os.getcwd())
644
- dirs = os.listdir(os.getcwd())
645
- import pprint
646
-
647
- pp = pprint.PrettyPrinter(indent=3)
648
- pp.pprint(dirs)
649
-
650
- # noinspection PyUnusedLocal
651
- @staticmethod
652
- def clear(args):
653
- if os.name == "nt":
654
- os.system('cls')
655
- else:
656
- os.system('clear')
657
-
658
- @staticmethod
659
- def typos(args):
660
- print('dlp: "{op}" is not an dlp command. Did you mean "{op}s"?'.format(op=args.operation))
661
-
662
-
663
- class Utils:
664
- def __init__(self, dl):
665
- self.dl = dl
666
-
667
- def get_packages_repo(self, args):
668
- if args.project_name is not None:
669
- project = self.dl.projects.get(project_name=args.project_name)
670
- packages = project.packages
671
- else:
672
- try:
673
- packages = self.dl.projects.get().packages
674
- except Exception:
675
- packages = self.dl.packages
676
-
677
- assert isinstance(packages, repositories.Packages)
678
- return packages
679
-
680
- def get_datasets_repo(self, args):
681
- if args.project_name is not None:
682
- project = self.dl.projects.get(project_name=args.project_name)
683
- datasets = project.datasets
684
- else:
685
- try:
686
- datasets = self.dl.projects.get().datasets
687
- except Exception:
688
- datasets = self.dl.datasets
689
-
690
- assert isinstance(datasets, repositories.Datasets)
691
- return datasets
692
-
693
- def get_services_repo(self, args):
694
- if args.project_name is not None:
695
- project = self.dl.projects.get(project_name=args.project_name)
696
- packages = project.packages
697
- services = project.services
698
- else:
699
- try:
700
- packages = self.dl.projects.get().packages
701
- services = self.dl.projects.get().services
702
- except Exception:
703
- packages = self.dl.packages
704
- services = self.dl.services
705
-
706
- if args.package_name is not None:
707
- package = packages.get(package_name=args.package_name)
708
- services = package.services
709
-
710
- assert isinstance(services, repositories.Services)
711
- return services
712
-
713
- def get_apps_repo(self, args) -> repositories.Apps:
714
- if args.project_name is not None:
715
- project = self.dl.projects.get(project_name=args.project_name)
716
- apps = project.apps
717
- else:
718
- try:
719
- apps = self.dl.projects.get().apps
720
- except Exception:
721
- apps = self.dl.apps
722
- assert isinstance(apps, repositories.Apps)
723
- return apps
724
-
725
- def get_dpks_repo(self, args) -> repositories.Dpks:
726
- if args.project_name is None and args.project_id is None:
727
- try:
728
- dpks = self.dl.projects.get().dpks
729
- except Exception:
730
- dpks = self.dl.dpks
731
- else:
732
- project = self.dl.projects.get(project_name=args.project_name, project_id=args.project_id)
733
- dpks = project.dpks
734
- if dpks.project is None:
735
- raise ValueError("Must input one of `project-name` or `project-id`") from None
736
- assert isinstance(dpks, repositories.Dpks)
737
- return dpks
738
-
739
- def get_triggers_repo(self, args):
740
- if args.project_name is not None:
741
- project = self.dl.projects.get(project_name=args.project_name)
742
- packages = project.packages
743
- services = project.services
744
- triggers = project.triggers
745
- else:
746
- try:
747
- packages = self.dl.projects.get().packages
748
- services = self.dl.projects.get().services
749
- triggers = self.dl.projects.get().triggers
750
- except Exception:
751
- packages = self.dl.packages
752
- services = self.dl.services
753
- triggers = self.dl.triggers
754
-
755
- if args.package_name is not None:
756
- package = packages.package_name(name=args.package_name)
757
- services = package.services
758
-
759
- if args.service_name is not None:
760
- service = services.get(service_name=args.service_name)
761
- triggers = service.triggers
762
-
763
- assert isinstance(triggers, repositories.Triggers)
764
- return triggers
1
+ import subprocess
2
+ import inquirer
3
+ import logging
4
+ import json
5
+ import os
6
+ import sys
7
+ import jwt
8
+
9
+ from .. import exceptions, entities, repositories, utilities, assets
10
+
11
+ logger = logging.getLogger(name='dtlpy')
12
+
13
+
14
+ class CommandExecutor:
15
+
16
+ def __init__(self, dl, parser):
17
+ self.dl = dl
18
+ self.parser = parser
19
+ self.utils = Utils(dl)
20
+
21
+ def run(self, args):
22
+ ########################
23
+ # Run Command if Exist #
24
+ ########################
25
+ if args.operation is None:
26
+ logger.info('See "dlp --help" for options')
27
+ return
28
+
29
+ operation = args.operation.lower().replace('-', '_')
30
+ if hasattr(self, operation):
31
+ getattr(self, operation)(args)
32
+ ###############
33
+ # Catch typos #
34
+ ###############
35
+ elif args.operation in ["project", 'dataset', 'item', 'service', 'package', 'video', 'deploy', 'generate']:
36
+ self.typos(args=args)
37
+ #######################
38
+ # Catch other options #
39
+ #######################
40
+ elif args.operation:
41
+ print('dlp: "%s" is not an dlp command' % args.operation)
42
+ print('See "dlp --help" for options')
43
+ else:
44
+ print('See "dlp --help" for options')
45
+
46
+ def help(self, args):
47
+ self.parser.print_help()
48
+
49
+ def logout(self, args):
50
+ self.dl.logout()
51
+ logger.info('logout successful')
52
+
53
+ # noinspection PyUnusedLocal
54
+ def login(self, args):
55
+ self.dl.login()
56
+ self.dl.info(with_token=False)
57
+
58
+ def login_token(self, args):
59
+ self.dl.login_token(args.token)
60
+ self.dl.info(with_token=False)
61
+
62
+ def login_api_key(self, args):
63
+ self.dl.login_api_key(api_key=args.api_key)
64
+
65
+ def login_secret(self, args):
66
+ self.login_m2m(args=args)
67
+
68
+ def login_m2m(self, args):
69
+ self.dl.login_m2m(email=args.email,
70
+ password=args.password,
71
+ client_id=args.client_id,
72
+ client_secret=args.client_secret)
73
+ self.dl.info(with_token=False)
74
+
75
+ def upgrade(self, args):
76
+ url = 'dtlpy'
77
+ if args.url is None:
78
+ try:
79
+ payload = jwt.decode(self.dl.client_api.token, algorithms=['HS256'],
80
+ verify=False, options={'verify_signature': False})
81
+ if 'admin' in payload['https://dataloop.ai/authorization']['roles']:
82
+ url = "https://storage.googleapis.com/dtlpy/dev/dtlpy-latest-py3-none-any.whl"
83
+ except Exception:
84
+ pass
85
+ else:
86
+ url = args.url
87
+
88
+ logger.info("Update DTLPy from {}".format(url))
89
+ logger.info("Installing using pip...")
90
+ cmd = "pip install {} --upgrade".format(url)
91
+ subprocess.Popen(cmd, shell=True)
92
+ sys.exit(0)
93
+
94
+ # noinspection PyUnusedLocal
95
+ def init(self, args):
96
+ self.dl.init()
97
+
98
+ # noinspection PyUnusedLocal
99
+ def checkout_state(self, args):
100
+ state = self.dl.checkout_state()
101
+ logger.info('Checked-out:')
102
+ for key, val in state.items():
103
+ try:
104
+ msg = '{entity} name: {name}\t{entity} id: {id}'.format(entity=key, name=val['name'], id=val['id'])
105
+ except KeyError:
106
+ msg = '{entity} Not Found'.format(entity=key)
107
+ logger.info(msg)
108
+
109
+ # noinspection PyUnusedLocal
110
+ def version(self, args):
111
+ logger.info("Dataloop SDK Version: {}".format(self.dl.__version__))
112
+
113
+ def development(self, args):
114
+ if args.development == "local":
115
+ # start the local development
116
+ if args.local == 'start':
117
+ development = self.utils.dl.client_api.state_io.get('development')
118
+ # create default values
119
+ if development is None:
120
+ development = dict()
121
+ if 'port' not in development:
122
+ development['port'] = 5802
123
+ if 'docker_image' not in development:
124
+ development['docker_image'] = 'dataloopai/dtlpy-agent:1.57.3.gpu.cuda11.5.py3.8.opencv'
125
+
126
+ # get values from input
127
+ if args.docker_image is not None:
128
+ development['docker_image'] = args.docker_image
129
+ if args.port is not None:
130
+ development['port'] = int(args.port)
131
+
132
+ # set values to local state
133
+ self.utils.dl.client_api.state_io.put('development', development)
134
+ utilities.local_development.start_session()
135
+ elif args.local == 'pause':
136
+ utilities.local_development.pause_session()
137
+ elif args.local == 'stop':
138
+ utilities.local_development.stop_session()
139
+ else:
140
+ print('Must select one of "start", "pause", "stop". Type "dlp development start --help" for options')
141
+ elif args.development == "remote":
142
+ logger.warning('FUTURE! This is not supported yet..')
143
+ else:
144
+ print('Type "dlp development --help" for options')
145
+
146
+ # noinspection PyUnusedLocal
147
+
148
+ def api(self, args):
149
+ if args.api == "info":
150
+ information = self.dl.info()
151
+ logger.info('-- Dataloop info --')
152
+ _ = [logger.info('{}: {}'.format(key, val)) for key, val in information.items()]
153
+
154
+ if args.api == "setenv":
155
+ self.dl.setenv(args.env)
156
+ logger.info("Platform environment: {}".format(self.dl.environment()))
157
+
158
+ def projects(self, args):
159
+ if args.projects == "ls":
160
+ self.dl.projects.list().print()
161
+ elif args.projects == "web":
162
+ if args.project_name is None:
163
+ args.project_name = self.dl.projects.get().name
164
+ self.dl.projects.open_in_web(project_name=args.project_name)
165
+ elif args.projects == "create":
166
+ project = self.dl.projects.create(args.project_name)
167
+ project.print()
168
+ project.checkout()
169
+ elif args.projects == "checkout":
170
+ self.dl.projects.checkout(project_name=args.project_name)
171
+ else:
172
+ print('Type "dlp projects --help" for options')
173
+
174
+ def datasets(self, args):
175
+ if args.datasets == "ls":
176
+ self.utils.get_datasets_repo(args=args).list().print()
177
+
178
+ elif args.datasets == "checkout":
179
+ self.dl.datasets.checkout(dataset_name=args.dataset_name)
180
+
181
+ elif args.datasets == "web":
182
+ if args.dataset_name is None:
183
+ args.dataset_name = self.dl.datasets.get().name
184
+ self.utils.get_datasets_repo(args=args).open_in_web(dataset_name=args.dataset_name)
185
+
186
+ elif args.datasets == "create":
187
+ dataset = self.utils.get_datasets_repo(args=args).create(dataset_name=args.dataset_name)
188
+ dataset.print()
189
+ if args.checkout:
190
+ dataset.checkout()
191
+ else:
192
+ print('Type "dlp datasets --help" for options')
193
+
194
+ def items(self, args):
195
+ if self.dl.token_expired():
196
+ logger.error("token expired, please login.")
197
+ return
198
+
199
+ if args.items == "ls":
200
+ project = self.dl.projects.get(project_name=args.project_name)
201
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
202
+ if isinstance(args.page, str):
203
+ try:
204
+ args.page = int(args.page)
205
+ except ValueError:
206
+ raise ValueError("Input --page must be integer")
207
+ filters = self.dl.Filters()
208
+
209
+ # add filters
210
+ if args.remote_path is not None:
211
+ if isinstance(args.remote_path, list):
212
+ filters.add(field="filename", values=args.remote_path, operator=entities.FiltersOperations.IN)
213
+ else:
214
+ filters.add(field="filename", values=args.remote_path)
215
+ if args.type is not None:
216
+ if isinstance(args.type, list):
217
+ filters.add(field='metadata.system.mimetype', values=args.type,
218
+ operator=entities.FiltersOperations.IN)
219
+ else:
220
+ filters.add(field="metadata.system.mimetype", values=args.type)
221
+
222
+ pages = dataset.items.list(filters=filters, page_offset=args.page)
223
+ pages.print()
224
+ print("Displaying page %d/%d" % (args.page, pages.total_pages_count - 1))
225
+
226
+ elif args.items == "web":
227
+ project = self.dl.projects.get(project_name=args.project_name)
228
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
229
+ dataset.items.open_in_web(filepath=args.remote_path)
230
+
231
+ elif args.items == "upload":
232
+ logger.info("Uploading directory...")
233
+ if isinstance(args.file_types, str):
234
+ args.file_types = [t.strip() for t in args.file_types.split(",")]
235
+ project = self.dl.projects.get(project_name=args.project_name)
236
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
237
+
238
+ dataset.items.upload(local_path=args.local_path,
239
+ remote_path=args.remote_path,
240
+ file_types=args.file_types,
241
+ overwrite=args.overwrite,
242
+ local_annotations_path=args.local_annotations_path)
243
+
244
+ elif args.items == "download":
245
+ logger.info("Downloading dataset...")
246
+ project = self.dl.projects.get(project_name=args.project_name)
247
+ dataset = project.datasets.get(dataset_name=args.dataset_name)
248
+
249
+ annotation_options = None
250
+ if args.annotation_options is not None:
251
+ annotation_options = [t.strip() for t in args.annotation_options.split(",")]
252
+
253
+ annotation_filters = None
254
+ if args.annotation_filter_type is not None or args.annotation_filter_label is not None:
255
+ annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
256
+ if args.annotation_filter_type is not None:
257
+ annotation_filter_type = [t.strip() for t in args.annotation_filter_type.split(",")]
258
+ annotation_filters.add(field='type',
259
+ values=annotation_filter_type,
260
+ operator=entities.FiltersOperations.IN)
261
+ if args.annotation_filter_label is not None:
262
+ annotation_filter_label = [t.strip() for t in args.annotation_filter_label.split(",")]
263
+ annotation_filters.add(field='label',
264
+ values=annotation_filter_label,
265
+ operator=entities.FiltersOperations.IN)
266
+ # create remote path filters
267
+ filters = self.dl.Filters()
268
+ if args.remote_path is not None:
269
+ remote_path = [t.strip() for t in args.remote_path.split(",")]
270
+ if len(remote_path) == 1:
271
+ remote_path = remote_path[0]
272
+ filters.add(field="filename", values=remote_path)
273
+ elif len(remote_path) > 1:
274
+ for item in remote_path:
275
+ if '*' in item:
276
+ filters.add(field="dir", values=item, method='or')
277
+ remote_path.pop(remote_path.index(item))
278
+ filters.add(field="dir", values=remote_path, operator=entities.FiltersOperations.IN, method='or')
279
+
280
+ if not args.without_binaries:
281
+ dataset.items.download(filters=filters,
282
+ local_path=args.local_path,
283
+ annotation_options=annotation_options,
284
+ annotation_filters=annotation_filters,
285
+ overwrite=args.overwrite,
286
+ with_text=args.with_text,
287
+ thickness=int(args.thickness),
288
+ to_items_folder=not args.not_items_folder)
289
+ else:
290
+ dataset.download_annotations(filters=filters,
291
+ local_path=args.local_path,
292
+ annotation_options=annotation_options,
293
+ annotation_filters=annotation_filters,
294
+ overwrite=args.overwrite,
295
+ with_text=args.with_text,
296
+ thickness=int(args.thickness))
297
+
298
+ else:
299
+ print('Type "dlp items --help" for options')
300
+
301
+ def videos(self, args):
302
+ if self.dl.token_expired():
303
+ print("[ERROR] token expired, please login.")
304
+ return
305
+
306
+ if args.videos == "play":
307
+ from dtlpy.utilities.videos.video_player import VideoPlayer
308
+
309
+ VideoPlayer(
310
+ item_filepath=args.item_path,
311
+ dataset_name=args.dataset_name,
312
+ project_name=args.project_name,
313
+ )
314
+
315
+ elif args.videos == "upload":
316
+ if (
317
+ (args.split_chunks is not None)
318
+ or (args.split_seconds is not None)
319
+ or (args.split_times is not None)
320
+ ):
321
+ # upload with split
322
+ if isinstance(args.split_chunks, str):
323
+ args.split_chunks = int(args.split_chunks)
324
+ if isinstance(args.split_seconds, str):
325
+ args.split_seconds = int(args.split_seconds)
326
+ if isinstance(args.split_times, str):
327
+ args.split_times = [int(sec) for sec in args.split_times.split(",")]
328
+ self.dl.utilities.videos.Videos.split_and_upload(
329
+ project_name=args.project_name,
330
+ dataset_name=args.dataset_name,
331
+ filepath=args.filename,
332
+ remote_path=args.remote_path,
333
+ split_chunks=args.split_chunks,
334
+ split_seconds=args.split_seconds,
335
+ split_pairs=args.split_times,
336
+ )
337
+ else:
338
+ print('Type "dlp videos --help" for options')
339
+
340
+ def services(self, args):
341
+ if self.dl.token_expired():
342
+ print("[ERROR] token expired, please login.")
343
+ return
344
+
345
+ elif args.services == "delete":
346
+ service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
347
+ service.delete()
348
+ logger.info('Service: "{}" deleted successfully'.format(service.name))
349
+
350
+ elif args.services == "ls":
351
+ self.utils.get_services_repo(args=args).list().print()
352
+ elif args.services == "log":
353
+ project = self.dl.projects.get(project_name=args.project_name)
354
+ service = project.services.get(service_name=args.service_name)
355
+
356
+ logs = service.log(start=args.start)
357
+ try:
358
+ for log in logs:
359
+ if isinstance(log, list):
360
+ for log_record in log:
361
+ print(log_record)
362
+ else:
363
+ print(log)
364
+ except KeyboardInterrupt:
365
+ pass
366
+
367
+ elif args.services == "execute":
368
+ try:
369
+ if args.service_name:
370
+ service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
371
+ else:
372
+ service = self.dl.services.get()
373
+ except Exception:
374
+ logger.info('Please checkout a service')
375
+ return
376
+
377
+ if args.annotation_id is not None:
378
+ resource = 'Annotation'
379
+ elif args.item_id is not None:
380
+ resource = 'Item'
381
+ elif args.dataset_id is not None:
382
+ resource = 'Dataset'
383
+ else:
384
+ resource = None
385
+
386
+ try:
387
+ execution_input = json.loads(args.inputs)
388
+ except Exception:
389
+ logger.info('Input should be json serializable')
390
+ return
391
+ if len(execution_input) == 0:
392
+ execution_input = None
393
+
394
+ logger.info('executing')
395
+ service.execute(sync=not args.asynchronous,
396
+ execution_input=execution_input,
397
+ function_name=args.function_name,
398
+ resource=resource,
399
+ item_id=args.item_id,
400
+ dataset_id=args.dataset_id,
401
+ annotation_id=args.annotation_id)
402
+ logger.info("Successfully executed function: {}".format(args.function_name))
403
+
404
+ else:
405
+ logger.info('Type "dlp packages --help" for options')
406
+
407
+ def deploy(self, args):
408
+ project = self.dl.projects.get(project_name=args.project_name)
409
+ json_filepath = args.json_file
410
+ deployed_services, package = self.dl.packages.deploy_from_file(project=project, json_filepath=json_filepath)
411
+ logger.info("Successfully deployed {} from file: {}\nServices: {}".format(len(deployed_services),
412
+ json_filepath,
413
+ [s.name for s in deployed_services]))
414
+
415
+ def generate(self, args):
416
+ package_type = args.package_type if args.package_type else self.dl.PackageCatalog.DEFAULT_PACKAGE_TYPE
417
+ self.dl.packages.generate(name=args.package_name, src_path=os.getcwd(), package_type=package_type)
418
+ self.utils.dl.client_api.state_io.put('package', {'name': args.package_name})
419
+ logger.info('Successfully generated package files')
420
+
421
+ def triggers(self, args):
422
+
423
+ if args.triggers == "create":
424
+ args.actions = [t.strip() for t in args.actions.split(",")]
425
+ try:
426
+ service = self.utils.get_services_repo(args).get(service_name=args.service_name)
427
+ except exceptions.NotFound:
428
+ logger.critical('Service not found. Please check-out a service or provide valid service name')
429
+ return
430
+
431
+ trigger = service.triggers.create(name=args.name,
432
+ filters=json.loads('{}'.format(args.filters.replace("'", '"'))),
433
+ function_name=args.function_name,
434
+ resource=args.resource,
435
+ actions=args.actions)
436
+ logger.info('Trigger created successfully: {}'.format(trigger.name))
437
+
438
+ elif args.triggers == "delete":
439
+ triggers = self.utils.get_triggers_repo(args=args)
440
+ triggers.get(trigger_name=args.trigger_name).delete()
441
+ logger.info('Trigger deleted successfully: {}'.format(args.trigger_name))
442
+ elif args.triggers == "ls":
443
+ triggers = self.utils.get_triggers_repo(args=args)
444
+ triggers.list().print()
445
+ else:
446
+ logger.info('Type "dlp packages --help" for options')
447
+
448
+ def packages(self, args):
449
+ if self.dl.token_expired():
450
+ logger.error("token expired, please login.")
451
+ return
452
+
453
+ elif args.packages == "delete":
454
+ package = self.utils.get_packages_repo(args=args).get(package_name=args.package_name)
455
+ package.delete()
456
+ logger.info('Successfully deleted package {}'.format(package.name))
457
+
458
+ elif args.packages == "ls":
459
+ self.utils.get_packages_repo(args=args).list().print()
460
+
461
+ elif args.packages == "checkout":
462
+ self.dl.packages.checkout(package_name=args.package_name)
463
+
464
+ elif args.packages == "push":
465
+ packages = self.utils.get_packages_repo(args=args)
466
+
467
+ package = packages.push(src_path=args.src_path,
468
+ package_name=args.package_name,
469
+ checkout=args.checkout)
470
+
471
+ logger.info("Successfully pushed package to platform\n"
472
+ "Package id:{}\nPackage version:{}".format(package.id,
473
+ package.version))
474
+ elif args.packages == "deploy":
475
+ packages = self.utils.get_packages_repo(args=args)
476
+
477
+ package = packages.deploy(package_name=args.package_name,
478
+ checkout=args.checkout,
479
+ module_name=args.module_name)
480
+
481
+ logger.info("Successfully pushed package to platform\n"
482
+ "Package id:{}\nPackage version:{}".format(package.id,
483
+ package.version))
484
+
485
+ elif args.packages == "test":
486
+ go_back = False
487
+ if 'src' in os.listdir(os.getcwd()):
488
+ go_back = True
489
+ os.chdir('./src')
490
+ try:
491
+ self.dl.packages.test_local_package(concurrency=int(args.concurrency),
492
+ function_name=args.function_name)
493
+ except Exception:
494
+ logger.exception('failed during test')
495
+ finally:
496
+ if go_back:
497
+ os.chdir('..')
498
+
499
+ else:
500
+ logger.info('Type "dlp packages --help" for options')
501
+
502
+ def app(self, args):
503
+ if args.app == 'pack':
504
+ path = self.dl.dpks.pack()
505
+ logger.info(f'Packed to {path}')
506
+ elif args.app == 'init':
507
+ app_filename = assets.paths.APP_JSON_FILENAME
508
+ if os.path.isfile(app_filename):
509
+ questions = [
510
+ inquirer.Confirm(name='overwrite',
511
+ message=f"Dataloop app already initialized. Re-initialize?",
512
+ default=False)]
513
+ answers = inquirer.prompt(questions)
514
+ if answers.get('overwrite') is False:
515
+ return
516
+ name = args.name
517
+ description = args.description
518
+ attributes = args.attributes
519
+ icon = args.icon
520
+ scope = args.scope
521
+ as_array = [name, description, attributes, icon, scope]
522
+ if as_array.count(None) == len(as_array): # No one value is initialized
523
+ dir_name = os.path.basename(os.getcwd())
524
+ questions = [
525
+ inquirer.Text(name='name',
526
+ message=f"Enter the name of the app (or press enter for '{dir_name}'):",
527
+ default=dir_name),
528
+ inquirer.Text(name='description',
529
+ message="Enter the description (or enter for empty): "),
530
+ inquirer.Text(name='attributes',
531
+ message="Enter the attributes (an object, or enter for empty): ",
532
+ default=None),
533
+ inquirer.Text(name='icon',
534
+ message="Enter the path to the icon (or enter for empty): "),
535
+ inquirer.Text(name='scope',
536
+ message="Enter the scope (or enter for 'organization'): ",
537
+ default='project'),
538
+ ]
539
+ answers = inquirer.prompt(questions)
540
+ name = answers.get('name')
541
+ description = answers.get('description')
542
+ attributes = answers.get('attributes')
543
+ icon = answers.get('icon')
544
+ scope = answers.get('scope')
545
+
546
+ if attributes is not None:
547
+ attributes = json.loads(attributes)
548
+
549
+ self.dl.dpks.init(name=name,
550
+ description=description,
551
+ attributes=attributes,
552
+ icon=icon,
553
+ scope=scope)
554
+ elif args.app == 'add':
555
+ if args.panel is True:
556
+ default_panel_name = "myPanel"
557
+ choices = list(entities.dpk.SlotType)
558
+
559
+ questions = [
560
+ inquirer.Text(name='name',
561
+ message=f"Enter PANEL NAME (or press enter for '{default_panel_name}'): ",
562
+ default='default_panel_name'),
563
+ inquirer.List(name='support_slot_type',
564
+ message="Enter SUPPORTED SLOT TYPE:",
565
+ choices=choices),
566
+ ]
567
+
568
+ answers = inquirer.prompt(questions)
569
+ #####
570
+ # create a dir for that panel
571
+ os.makedirs(answers.get('name'), exist_ok=True)
572
+ # dump to dataloop.json
573
+ app_filename = assets.paths.APP_JSON_FILENAME
574
+ if not os.path.isfile(app_filename):
575
+ logger.error(f"Can't find app config file ({app_filename}), please run `dlp app init` first")
576
+ else:
577
+ with open(app_filename, 'r') as f:
578
+ dpk = entities.Dpk.from_json(json.load(f))
579
+ dpk.components.panels.append(entities.Panel(name=answers.get('name'),
580
+ supported_slots=[answers.get('support_slot_type')]))
581
+ with open(app_filename, 'w') as f:
582
+ json.dump(dpk.to_json(), f, indent=4)
583
+
584
+ elif args.app == 'publish':
585
+ dpk = self.utils.get_dpks_repo(args).publish()
586
+ if dpk:
587
+ logger.info(f'Published the application: id={dpk.id}')
588
+ else:
589
+ logger.info("Couldn't publish the application")
590
+ elif args.app == 'update':
591
+ # TODO: I think it changed, not implemented
592
+ logger.info('App updated successfully')
593
+ elif args.app == 'resume':
594
+ succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
595
+ if succeed is True:
596
+ logger.info('Resumed application successfully')
597
+ else:
598
+ logger.info('Application resume failed')
599
+ elif args.app == 'pause':
600
+ succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
601
+ if succeed is True:
602
+ logger.info('Paused application successfully')
603
+ else:
604
+ logger.info('Application pause failed')
605
+ elif args.app == 'install':
606
+ app = self.utils.get_apps_repo(args).install(
607
+ dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
608
+ organization_id=args.org_id
609
+ )
610
+ if app is not None:
611
+ logger.info('App installed successfully')
612
+ else:
613
+ logger.info("Couldn't install the app")
614
+ elif args.app == 'pull':
615
+ succeed = self.dl.dpks.pull(dpk_name=args.dpk_name)
616
+ if succeed is True:
617
+ logger.info("Pulled successfully")
618
+ else:
619
+ logger.info("Couldn't pull")
620
+ elif args.app == 'list':
621
+ self.dl.apps.list().print()
622
+
623
+ # noinspection PyUnusedLocal
624
+ @staticmethod
625
+ def pwd(args):
626
+ print(os.getcwd())
627
+
628
+ @staticmethod
629
+ def cd(args):
630
+ directory = args.dir
631
+ if directory == '..':
632
+ directory = os.path.split(os.getcwd())[0]
633
+ os.chdir(directory)
634
+ print(os.getcwd())
635
+
636
+ @staticmethod
637
+ def mkdir(args):
638
+ os.mkdir(args.name)
639
+
640
+ # noinspection PyUnusedLocal
641
+ @staticmethod
642
+ def ls(args):
643
+ print(os.getcwd())
644
+ dirs = os.listdir(os.getcwd())
645
+ import pprint
646
+
647
+ pp = pprint.PrettyPrinter(indent=3)
648
+ pp.pprint(dirs)
649
+
650
+ # noinspection PyUnusedLocal
651
+ @staticmethod
652
+ def clear(args):
653
+ if os.name == "nt":
654
+ os.system('cls')
655
+ else:
656
+ os.system('clear')
657
+
658
+ @staticmethod
659
+ def typos(args):
660
+ print('dlp: "{op}" is not an dlp command. Did you mean "{op}s"?'.format(op=args.operation))
661
+
662
+
663
+ class Utils:
664
+ def __init__(self, dl):
665
+ self.dl = dl
666
+
667
+ def get_packages_repo(self, args):
668
+ if args.project_name is not None:
669
+ project = self.dl.projects.get(project_name=args.project_name)
670
+ packages = project.packages
671
+ else:
672
+ try:
673
+ packages = self.dl.projects.get().packages
674
+ except Exception:
675
+ packages = self.dl.packages
676
+
677
+ assert isinstance(packages, repositories.Packages)
678
+ return packages
679
+
680
+ def get_datasets_repo(self, args):
681
+ if args.project_name is not None:
682
+ project = self.dl.projects.get(project_name=args.project_name)
683
+ datasets = project.datasets
684
+ else:
685
+ try:
686
+ datasets = self.dl.projects.get().datasets
687
+ except Exception:
688
+ datasets = self.dl.datasets
689
+
690
+ assert isinstance(datasets, repositories.Datasets)
691
+ return datasets
692
+
693
+ def get_services_repo(self, args):
694
+ if args.project_name is not None:
695
+ project = self.dl.projects.get(project_name=args.project_name)
696
+ packages = project.packages
697
+ services = project.services
698
+ else:
699
+ try:
700
+ packages = self.dl.projects.get().packages
701
+ services = self.dl.projects.get().services
702
+ except Exception:
703
+ packages = self.dl.packages
704
+ services = self.dl.services
705
+
706
+ if args.package_name is not None:
707
+ package = packages.get(package_name=args.package_name)
708
+ services = package.services
709
+
710
+ assert isinstance(services, repositories.Services)
711
+ return services
712
+
713
+ def get_apps_repo(self, args) -> repositories.Apps:
714
+ if args.project_name is not None:
715
+ project = self.dl.projects.get(project_name=args.project_name)
716
+ apps = project.apps
717
+ else:
718
+ try:
719
+ apps = self.dl.projects.get().apps
720
+ except Exception:
721
+ apps = self.dl.apps
722
+ assert isinstance(apps, repositories.Apps)
723
+ return apps
724
+
725
+ def get_dpks_repo(self, args) -> repositories.Dpks:
726
+ if args.project_name is None and args.project_id is None:
727
+ try:
728
+ dpks = self.dl.projects.get().dpks
729
+ except Exception:
730
+ dpks = self.dl.dpks
731
+ else:
732
+ project = self.dl.projects.get(project_name=args.project_name, project_id=args.project_id)
733
+ dpks = project.dpks
734
+ if dpks.project is None:
735
+ raise ValueError("Must input one of `project-name` or `project-id`") from None
736
+ assert isinstance(dpks, repositories.Dpks)
737
+ return dpks
738
+
739
+ def get_triggers_repo(self, args):
740
+ if args.project_name is not None:
741
+ project = self.dl.projects.get(project_name=args.project_name)
742
+ packages = project.packages
743
+ services = project.services
744
+ triggers = project.triggers
745
+ else:
746
+ try:
747
+ packages = self.dl.projects.get().packages
748
+ services = self.dl.projects.get().services
749
+ triggers = self.dl.projects.get().triggers
750
+ except Exception:
751
+ packages = self.dl.packages
752
+ services = self.dl.services
753
+ triggers = self.dl.triggers
754
+
755
+ if args.package_name is not None:
756
+ package = packages.package_name(name=args.package_name)
757
+ services = package.services
758
+
759
+ if args.service_name is not None:
760
+ service = services.get(service_name=args.service_name)
761
+ triggers = service.triggers
762
+
763
+ assert isinstance(triggers, repositories.Triggers)
764
+ return triggers