dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,367 +1,367 @@
1
- from prompt_toolkit.completion import Completer, Completion
2
- from prompt_toolkit.history import History
3
- from fuzzyfinder.main import fuzzyfinder
4
- import csv
5
- import threading
6
- import os
7
- import datetime
8
- import logging
9
-
10
-
11
- class StateEnum:
12
- """
13
- State enum
14
- """
15
- START = 0
16
- RUNNING = 1
17
- DONE = 2
18
- CONTINUE = 3
19
-
20
-
21
- def get_parser_tree(parser):
22
- """
23
- Creates parser tree for autocomplete
24
- :param parser: parser
25
- :return: parser tree
26
- """
27
- if parser._subparsers is None:
28
- p_keywords = list()
29
- for param in parser._option_string_actions:
30
- if param.startswith('--'):
31
- p_keywords.append(param)
32
- else:
33
- p_keywords = dict()
34
- subparsers = parser._subparsers._group_actions[0].choices
35
- for sub_parser in subparsers:
36
- p_keywords[sub_parser] = get_parser_tree(subparsers[sub_parser])
37
- if 'shell' in p_keywords:
38
- # noinspection PyTypeChecker
39
- p_keywords.pop('shell')
40
- return p_keywords
41
-
42
-
43
- class FileHistory(History):
44
- """
45
- :class:`.History` class that stores all strings in a file.
46
- """
47
-
48
- def __init__(self, filename):
49
- self.filename = filename
50
- super(FileHistory, self).__init__()
51
- self.to_hide = ['password', 'secret']
52
-
53
- def load_history_strings(self):
54
- strings = []
55
- lines = []
56
-
57
- def add():
58
- if lines:
59
- # Join and drop trailing newline.
60
- string = ''.join(lines)[:-1]
61
- hide = any(field in string for field in self.to_hide)
62
- if not hide:
63
- strings.append(string)
64
-
65
- if os.path.exists(self.filename):
66
- with open(self.filename, 'rb') as f:
67
- for line in f:
68
- line = line.decode('utf-8')
69
-
70
- if line.startswith('+'):
71
- lines.append(line[1:])
72
- else:
73
- add()
74
- lines = []
75
-
76
- add()
77
-
78
- # Reverse the order, because newest items have to go first.
79
- return reversed(strings)
80
-
81
- def store_string(self, string):
82
- # Save to file.
83
- with open(self.filename, 'ab') as f:
84
- def write(t):
85
- f.write(t.encode('utf-8'))
86
-
87
- write('\n# %s\n' % datetime.datetime.now(datetime.timezone.utc))
88
- for line in string.split('\n'):
89
- hide = any(field in line for field in self.to_hide)
90
- if not hide:
91
- write('+%s\n' % line)
92
-
93
-
94
- class DlpCompleter(Completer):
95
- """
96
- Autocomplete for dlp shell
97
- """
98
-
99
- def __init__(self, keywords, dlp):
100
- super(DlpCompleter, self).__init__()
101
- # globals
102
- self.keywords = keywords
103
- self.param_suggestions = list()
104
- self.thread_state = StateEnum.START
105
- self.dlp = dlp
106
-
107
- def get_param_suggestions(self, param, word_before_cursor, cmd):
108
- """
109
- Return parap suggestions
110
- :param param:
111
- :param word_before_cursor:
112
- :param cmd:
113
- :return:
114
- """
115
- prev_state = logging.root.manager.disable
116
- try:
117
- logging.root.manager.disable = logging.ERROR
118
- if self.thread_state in [StateEnum.RUNNING, StateEnum.DONE]:
119
- return
120
- else:
121
- if param == '--project-name':
122
- self.thread_state = StateEnum.RUNNING
123
- project_list = self.dlp.projects.list()
124
- self.param_suggestions = ['"{}'.format(project.name) for project in project_list]
125
- self.thread_state = StateEnum.DONE
126
-
127
- elif param == '--package-name':
128
- self.thread_state = StateEnum.RUNNING
129
- if '--project-name' in cmd:
130
- project = self.dlp.projects.get(
131
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
132
- packages = project.packages
133
- else:
134
- packages = self.dlp.packages
135
- package_list = packages.list()
136
- self.param_suggestions = ['"{}'.format(package.name) for package in package_list.items]
137
- self.thread_state = StateEnum.DONE
138
-
139
- elif param == '--service-name':
140
- self.thread_state = StateEnum.RUNNING
141
- if '--package-name' in cmd:
142
- package = self.dlp.packages.get(
143
- package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
144
- services = package.services
145
- elif '--project-name' in cmd:
146
- project = self.dlp.projects.get(
147
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
148
- services = project.services
149
- else:
150
- services = self.dlp.services
151
-
152
- service_list = services.list()
153
- self.param_suggestions = ['"{}'.format(service.name) for service in service_list.items]
154
- self.thread_state = StateEnum.DONE
155
-
156
- elif param == '--trigger-name':
157
- self.thread_state = StateEnum.RUNNING
158
- if '--service-name' in cmd:
159
- service = self.dlp.services.get(
160
- service_name=cmd[cmd.index('--service-name') + 1].replace('"', ''))
161
- triggers = service.triggers
162
- elif '--package-name' in cmd:
163
- package = self.dlp.packages.get(
164
- package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
165
- triggers = package.services
166
- elif '--project-name' in cmd:
167
- project = self.dlp.projects.get(
168
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
169
- triggers = project.services
170
- else:
171
- triggers = self.dlp.services
172
-
173
- trigger_list = triggers.list()
174
- self.param_suggestions = ['"{}'.format(trigger.name) for trigger in trigger_list.items]
175
- self.thread_state = StateEnum.DONE
176
-
177
- elif param == '--dataset-name':
178
- self.thread_state = StateEnum.RUNNING
179
- if '--project-name' in cmd:
180
- project = self.dlp.projects.get(
181
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
182
- dataset_list = project.datasets.list()
183
- else:
184
- project = self.dlp.projects.get()
185
- dataset_list = project.datasets.list()
186
- self.param_suggestions = ['"{}'.format(dataset.name) for dataset in dataset_list]
187
- self.thread_state = StateEnum.DONE
188
-
189
- elif param == '--remote-path':
190
- self.thread_state = StateEnum.RUNNING
191
- if '--project-name' in cmd:
192
- project = self.dlp.projects.get(
193
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
194
- else:
195
- project = self.dlp.projects.get()
196
- if '--dataset-name' in cmd:
197
- dataset = project.datasets.get(
198
- dataset_name=cmd[cmd.index('--dataset-name') + 1].replace('"', ''))
199
- else:
200
- dataset = self.dlp.datasets.get()
201
- self.param_suggestions = dataset.directory_tree.dir_names
202
- with_quotation = ['"{}'.format(directory) for directory in dataset.directory_tree.dir_names]
203
- self.param_suggestions += with_quotation
204
- self.thread_state = StateEnum.DONE
205
-
206
- elif param in ['--local-path', '--local-annotations-path', '--service-file']:
207
- self.thread_state = StateEnum.CONTINUE
208
- param = word_before_cursor.replace('"', '')
209
- if param == '':
210
- param, path = os.path.splitdrive(os.getcwd())
211
- param += os.path.sep
212
- self.param_suggestions = ['"{}'.format(os.path.join(param, directory))
213
- for directory in os.listdir(param)
214
- if not directory.startswith('.')]
215
- elif os.path.isdir(os.path.dirname(param)):
216
- base_dir = os.path.dirname(param)
217
- self.param_suggestions = ['"{}'.format(os.path.join(base_dir, directory))
218
- for directory in os.listdir(base_dir)
219
- if (not directory.startswith('.') and
220
- param in '"{}'.format(os.path.join(base_dir, directory)))]
221
-
222
- elif param in ['--annotation-options']:
223
- self.thread_state = StateEnum.CONTINUE
224
- self.param_suggestions = ['mask', 'json', 'instance', '"mask, json"',
225
- '"mask, instance"', '"json, instance"', '"mask, json, instance"']
226
-
227
- elif param in ['--annotation-filter-type']:
228
- self.thread_state = StateEnum.CONTINUE
229
- self.param_suggestions = ['box', 'segment', 'binary', '"box, segment"']
230
-
231
- elif param in ['--annotation-filter-label']:
232
- self.thread_state = StateEnum.CONTINUE
233
-
234
- elif param in ['--actions']:
235
- self.thread_state = StateEnum.CONTINUE
236
- self.param_suggestions = ['Created', 'Updated', 'Deleted', '"Created, Updated"',
237
- '"Created, Deleted"', '"Updated, Deleted"', '"Created, Updated, Deleted"']
238
-
239
- elif param in ['--resource']:
240
- self.thread_state = StateEnum.CONTINUE
241
- self.param_suggestions = [value for key, value in self.dlp.TriggerResource.__dict__.items() if
242
- not key.startswith('_')]
243
-
244
- elif param in ['--package-type']:
245
- self.thread_state = StateEnum.CONTINUE
246
- self.param_suggestions = [value for key, value in self.dlp.PackageCatalog.__dict__.items() if
247
- not key.startswith('_')]
248
-
249
- # elif param in ['cd']:
250
- # self.thread_state = StateEnum.CONTINUE
251
- # if word_before_cursor != '' and os.path.isdir(os.path.join(os.getcwd(), word_before_cursor)):
252
- # self.param_suggestions = [os.path.join(word_before_cursor, directory)
253
- # for directory in
254
- # os.listdir(os.path.join(os.getcwd(), word_before_cursor))
255
- # if os.path.isdir(
256
- # os.path.join(os.getcwd(), word_before_cursor, directory))]
257
- # else:
258
- # self.param_suggestions = [directory for directory in os.listdir(os.getcwd()) if
259
- # os.path.isdir(directory)]
260
-
261
- else:
262
- self.thread_state = StateEnum.START
263
- self.param_suggestions = list()
264
- except Exception:
265
- self.param_suggestions = list()
266
- self.thread_state = StateEnum.START
267
- finally:
268
- logging.root.manager.disable = prev_state
269
-
270
- def need_param(self, cmd, word_before_cursor):
271
- """
272
- :param cmd:
273
- :param word_before_cursor:
274
- """
275
- need_param = False
276
-
277
- try:
278
- bool_flags_list = ['--overwrite', '--with-text', '--deploy', '--not-items-folder', '--encode',
279
- '--without-binaries']
280
-
281
- if len(cmd) > 2 and cmd[-2].startswith('--') and word_before_cursor != '':
282
- need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
283
- elif cmd[-1].startswith('-') and word_before_cursor == '':
284
- need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
285
- elif cmd[0] in ['cd']:
286
- need_param = True
287
- except Exception:
288
- need_param = False
289
-
290
- return need_param
291
-
292
- @staticmethod
293
- def get_param(cmd, word_before_cursor):
294
- """
295
- :param cmd:
296
- :param word_before_cursor:
297
- """
298
- if word_before_cursor == '' or len(cmd) < 2:
299
- param = cmd[-1]
300
- else:
301
- param = cmd[-2]
302
-
303
- return param
304
-
305
- def get_completions(self, document, complete_event):
306
- """
307
- Get command completion
308
- :param document:
309
- :param complete_event:
310
- :return:
311
- """
312
- # fix input
313
- cmd = next(csv.reader([" ".join(document.text.split())], delimiter=' '))
314
-
315
- # get current word
316
- word_before_cursor = document.get_word_before_cursor(WORD=True)
317
-
318
- # suggest keywords
319
- suggestions = list()
320
- if self.need_param(cmd=cmd, word_before_cursor=word_before_cursor):
321
- param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor)
322
- if self.thread_state in [StateEnum.START, StateEnum.CONTINUE]:
323
- if param in ['--project-name',
324
- '--dataset-name',
325
- '--remote-path',
326
- '--package-name',
327
- '--service-name',
328
- '--trigger-name']:
329
- thread = threading.Thread(target=self.get_param_suggestions,
330
- kwargs={"param": param,
331
- 'word_before_cursor': word_before_cursor,
332
- 'cmd': cmd})
333
- thread.daemon = True
334
- thread.start()
335
-
336
- else:
337
- self.get_param_suggestions(param=param, word_before_cursor=word_before_cursor, cmd=cmd)
338
- if self.thread_state in [StateEnum.DONE, StateEnum.CONTINUE]:
339
- suggestions = self.param_suggestions
340
- elif len(cmd) == 0:
341
- suggestions = list(self.keywords.keys())
342
- elif len(cmd) == 1:
343
- self.thread_state = StateEnum.START
344
- if cmd[0] not in self.keywords.keys() and cmd[0] != '':
345
- if not word_before_cursor == '':
346
- suggestions = list(self.keywords.keys())
347
- elif cmd[0] == '':
348
- suggestions = list(self.keywords.keys())
349
- elif isinstance(self.keywords[cmd[0]], list):
350
- suggestions = self.keywords[cmd[0]]
351
- elif isinstance(self.keywords[cmd[0]], dict):
352
- suggestions = list(self.keywords[cmd[0]].keys())
353
- elif len(cmd) >= 2:
354
- self.thread_state = StateEnum.START
355
- if cmd[0] not in self.keywords.keys():
356
- suggestions = list()
357
- elif isinstance(self.keywords[cmd[0]], list):
358
- suggestions = self.keywords[cmd[0]]
359
- elif isinstance(self.keywords[cmd[0]], dict):
360
- if cmd[1] in self.keywords[cmd[0]].keys():
361
- suggestions = self.keywords[cmd[0]][cmd[1]]
362
- else:
363
- suggestions = list(self.keywords[cmd[0]].keys())
364
-
365
- matches = fuzzyfinder(word_before_cursor, suggestions)
366
- for match in matches:
367
- yield Completion(match, start_position=-len(word_before_cursor))
1
+ from prompt_toolkit.completion import Completer, Completion
2
+ from prompt_toolkit.history import History
3
+ from fuzzyfinder.main import fuzzyfinder
4
+ import csv
5
+ import threading
6
+ import os
7
+ import datetime
8
+ import logging
9
+
10
+
11
+ class StateEnum:
12
+ """
13
+ State enum
14
+ """
15
+ START = 0
16
+ RUNNING = 1
17
+ DONE = 2
18
+ CONTINUE = 3
19
+
20
+
21
+ def get_parser_tree(parser):
22
+ """
23
+ Creates parser tree for autocomplete
24
+ :param parser: parser
25
+ :return: parser tree
26
+ """
27
+ if parser._subparsers is None:
28
+ p_keywords = list()
29
+ for param in parser._option_string_actions:
30
+ if param.startswith('--'):
31
+ p_keywords.append(param)
32
+ else:
33
+ p_keywords = dict()
34
+ subparsers = parser._subparsers._group_actions[0].choices
35
+ for sub_parser in subparsers:
36
+ p_keywords[sub_parser] = get_parser_tree(subparsers[sub_parser])
37
+ if 'shell' in p_keywords:
38
+ # noinspection PyTypeChecker
39
+ p_keywords.pop('shell')
40
+ return p_keywords
41
+
42
+
43
+ class FileHistory(History):
44
+ """
45
+ :class:`.History` class that stores all strings in a file.
46
+ """
47
+
48
+ def __init__(self, filename):
49
+ self.filename = filename
50
+ super(FileHistory, self).__init__()
51
+ self.to_hide = ['password', 'secret']
52
+
53
+ def load_history_strings(self):
54
+ strings = []
55
+ lines = []
56
+
57
+ def add():
58
+ if lines:
59
+ # Join and drop trailing newline.
60
+ string = ''.join(lines)[:-1]
61
+ hide = any(field in string for field in self.to_hide)
62
+ if not hide:
63
+ strings.append(string)
64
+
65
+ if os.path.exists(self.filename):
66
+ with open(self.filename, 'rb') as f:
67
+ for line in f:
68
+ line = line.decode('utf-8')
69
+
70
+ if line.startswith('+'):
71
+ lines.append(line[1:])
72
+ else:
73
+ add()
74
+ lines = []
75
+
76
+ add()
77
+
78
+ # Reverse the order, because newest items have to go first.
79
+ return reversed(strings)
80
+
81
+ def store_string(self, string):
82
+ # Save to file.
83
+ with open(self.filename, 'ab') as f:
84
+ def write(t):
85
+ f.write(t.encode('utf-8'))
86
+
87
+ write('\n# %s\n' % datetime.datetime.now(datetime.timezone.utc))
88
+ for line in string.split('\n'):
89
+ hide = any(field in line for field in self.to_hide)
90
+ if not hide:
91
+ write('+%s\n' % line)
92
+
93
+
94
+ class DlpCompleter(Completer):
95
+ """
96
+ Autocomplete for dlp shell
97
+ """
98
+
99
+ def __init__(self, keywords, dlp):
100
+ super(DlpCompleter, self).__init__()
101
+ # globals
102
+ self.keywords = keywords
103
+ self.param_suggestions = list()
104
+ self.thread_state = StateEnum.START
105
+ self.dlp = dlp
106
+
107
+ def get_param_suggestions(self, param, word_before_cursor, cmd):
108
+ """
109
+ Return parap suggestions
110
+ :param param:
111
+ :param word_before_cursor:
112
+ :param cmd:
113
+ :return:
114
+ """
115
+ prev_state = logging.root.manager.disable
116
+ try:
117
+ logging.root.manager.disable = logging.ERROR
118
+ if self.thread_state in [StateEnum.RUNNING, StateEnum.DONE]:
119
+ return
120
+ else:
121
+ if param == '--project-name':
122
+ self.thread_state = StateEnum.RUNNING
123
+ project_list = self.dlp.projects.list()
124
+ self.param_suggestions = ['"{}'.format(project.name) for project in project_list]
125
+ self.thread_state = StateEnum.DONE
126
+
127
+ elif param == '--package-name':
128
+ self.thread_state = StateEnum.RUNNING
129
+ if '--project-name' in cmd:
130
+ project = self.dlp.projects.get(
131
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
132
+ packages = project.packages
133
+ else:
134
+ packages = self.dlp.packages
135
+ package_list = packages.list()
136
+ self.param_suggestions = ['"{}'.format(package.name) for package in package_list.items]
137
+ self.thread_state = StateEnum.DONE
138
+
139
+ elif param == '--service-name':
140
+ self.thread_state = StateEnum.RUNNING
141
+ if '--package-name' in cmd:
142
+ package = self.dlp.packages.get(
143
+ package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
144
+ services = package.services
145
+ elif '--project-name' in cmd:
146
+ project = self.dlp.projects.get(
147
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
148
+ services = project.services
149
+ else:
150
+ services = self.dlp.services
151
+
152
+ service_list = services.list()
153
+ self.param_suggestions = ['"{}'.format(service.name) for service in service_list.items]
154
+ self.thread_state = StateEnum.DONE
155
+
156
+ elif param == '--trigger-name':
157
+ self.thread_state = StateEnum.RUNNING
158
+ if '--service-name' in cmd:
159
+ service = self.dlp.services.get(
160
+ service_name=cmd[cmd.index('--service-name') + 1].replace('"', ''))
161
+ triggers = service.triggers
162
+ elif '--package-name' in cmd:
163
+ package = self.dlp.packages.get(
164
+ package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
165
+ triggers = package.services
166
+ elif '--project-name' in cmd:
167
+ project = self.dlp.projects.get(
168
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
169
+ triggers = project.services
170
+ else:
171
+ triggers = self.dlp.services
172
+
173
+ trigger_list = triggers.list()
174
+ self.param_suggestions = ['"{}'.format(trigger.name) for trigger in trigger_list.items]
175
+ self.thread_state = StateEnum.DONE
176
+
177
+ elif param == '--dataset-name':
178
+ self.thread_state = StateEnum.RUNNING
179
+ if '--project-name' in cmd:
180
+ project = self.dlp.projects.get(
181
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
182
+ dataset_list = project.datasets.list()
183
+ else:
184
+ project = self.dlp.projects.get()
185
+ dataset_list = project.datasets.list()
186
+ self.param_suggestions = ['"{}'.format(dataset.name) for dataset in dataset_list]
187
+ self.thread_state = StateEnum.DONE
188
+
189
+ elif param == '--remote-path':
190
+ self.thread_state = StateEnum.RUNNING
191
+ if '--project-name' in cmd:
192
+ project = self.dlp.projects.get(
193
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
194
+ else:
195
+ project = self.dlp.projects.get()
196
+ if '--dataset-name' in cmd:
197
+ dataset = project.datasets.get(
198
+ dataset_name=cmd[cmd.index('--dataset-name') + 1].replace('"', ''))
199
+ else:
200
+ dataset = self.dlp.datasets.get()
201
+ self.param_suggestions = dataset.directory_tree.dir_names
202
+ with_quotation = ['"{}'.format(directory) for directory in dataset.directory_tree.dir_names]
203
+ self.param_suggestions += with_quotation
204
+ self.thread_state = StateEnum.DONE
205
+
206
+ elif param in ['--local-path', '--local-annotations-path', '--service-file']:
207
+ self.thread_state = StateEnum.CONTINUE
208
+ param = word_before_cursor.replace('"', '')
209
+ if param == '':
210
+ param, path = os.path.splitdrive(os.getcwd())
211
+ param += os.path.sep
212
+ self.param_suggestions = ['"{}'.format(os.path.join(param, directory))
213
+ for directory in os.listdir(param)
214
+ if not directory.startswith('.')]
215
+ elif os.path.isdir(os.path.dirname(param)):
216
+ base_dir = os.path.dirname(param)
217
+ self.param_suggestions = ['"{}'.format(os.path.join(base_dir, directory))
218
+ for directory in os.listdir(base_dir)
219
+ if (not directory.startswith('.') and
220
+ param in '"{}'.format(os.path.join(base_dir, directory)))]
221
+
222
+ elif param in ['--annotation-options']:
223
+ self.thread_state = StateEnum.CONTINUE
224
+ self.param_suggestions = ['mask', 'json', 'instance', '"mask, json"',
225
+ '"mask, instance"', '"json, instance"', '"mask, json, instance"']
226
+
227
+ elif param in ['--annotation-filter-type']:
228
+ self.thread_state = StateEnum.CONTINUE
229
+ self.param_suggestions = ['box', 'segment', 'binary', '"box, segment"']
230
+
231
+ elif param in ['--annotation-filter-label']:
232
+ self.thread_state = StateEnum.CONTINUE
233
+
234
+ elif param in ['--actions']:
235
+ self.thread_state = StateEnum.CONTINUE
236
+ self.param_suggestions = ['Created', 'Updated', 'Deleted', '"Created, Updated"',
237
+ '"Created, Deleted"', '"Updated, Deleted"', '"Created, Updated, Deleted"']
238
+
239
+ elif param in ['--resource']:
240
+ self.thread_state = StateEnum.CONTINUE
241
+ self.param_suggestions = [value for key, value in self.dlp.TriggerResource.__dict__.items() if
242
+ not key.startswith('_')]
243
+
244
+ elif param in ['--package-type']:
245
+ self.thread_state = StateEnum.CONTINUE
246
+ self.param_suggestions = [value for key, value in self.dlp.PackageCatalog.__dict__.items() if
247
+ not key.startswith('_')]
248
+
249
+ # elif param in ['cd']:
250
+ # self.thread_state = StateEnum.CONTINUE
251
+ # if word_before_cursor != '' and os.path.isdir(os.path.join(os.getcwd(), word_before_cursor)):
252
+ # self.param_suggestions = [os.path.join(word_before_cursor, directory)
253
+ # for directory in
254
+ # os.listdir(os.path.join(os.getcwd(), word_before_cursor))
255
+ # if os.path.isdir(
256
+ # os.path.join(os.getcwd(), word_before_cursor, directory))]
257
+ # else:
258
+ # self.param_suggestions = [directory for directory in os.listdir(os.getcwd()) if
259
+ # os.path.isdir(directory)]
260
+
261
+ else:
262
+ self.thread_state = StateEnum.START
263
+ self.param_suggestions = list()
264
+ except Exception:
265
+ self.param_suggestions = list()
266
+ self.thread_state = StateEnum.START
267
+ finally:
268
+ logging.root.manager.disable = prev_state
269
+
270
+ def need_param(self, cmd, word_before_cursor):
271
+ """
272
+ :param cmd:
273
+ :param word_before_cursor:
274
+ """
275
+ need_param = False
276
+
277
+ try:
278
+ bool_flags_list = ['--overwrite', '--with-text', '--deploy', '--not-items-folder', '--encode',
279
+ '--without-binaries']
280
+
281
+ if len(cmd) > 2 and cmd[-2].startswith('--') and word_before_cursor != '':
282
+ need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
283
+ elif cmd[-1].startswith('-') and word_before_cursor == '':
284
+ need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
285
+ elif cmd[0] in ['cd']:
286
+ need_param = True
287
+ except Exception:
288
+ need_param = False
289
+
290
+ return need_param
291
+
292
+ @staticmethod
293
+ def get_param(cmd, word_before_cursor):
294
+ """
295
+ :param cmd:
296
+ :param word_before_cursor:
297
+ """
298
+ if word_before_cursor == '' or len(cmd) < 2:
299
+ param = cmd[-1]
300
+ else:
301
+ param = cmd[-2]
302
+
303
+ return param
304
+
305
+ def get_completions(self, document, complete_event):
306
+ """
307
+ Get command completion
308
+ :param document:
309
+ :param complete_event:
310
+ :return:
311
+ """
312
+ # fix input
313
+ cmd = next(csv.reader([" ".join(document.text.split())], delimiter=' '))
314
+
315
+ # get current word
316
+ word_before_cursor = document.get_word_before_cursor(WORD=True)
317
+
318
+ # suggest keywords
319
+ suggestions = list()
320
+ if self.need_param(cmd=cmd, word_before_cursor=word_before_cursor):
321
+ param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor)
322
+ if self.thread_state in [StateEnum.START, StateEnum.CONTINUE]:
323
+ if param in ['--project-name',
324
+ '--dataset-name',
325
+ '--remote-path',
326
+ '--package-name',
327
+ '--service-name',
328
+ '--trigger-name']:
329
+ thread = threading.Thread(target=self.get_param_suggestions,
330
+ kwargs={"param": param,
331
+ 'word_before_cursor': word_before_cursor,
332
+ 'cmd': cmd})
333
+ thread.daemon = True
334
+ thread.start()
335
+
336
+ else:
337
+ self.get_param_suggestions(param=param, word_before_cursor=word_before_cursor, cmd=cmd)
338
+ if self.thread_state in [StateEnum.DONE, StateEnum.CONTINUE]:
339
+ suggestions = self.param_suggestions
340
+ elif len(cmd) == 0:
341
+ suggestions = list(self.keywords.keys())
342
+ elif len(cmd) == 1:
343
+ self.thread_state = StateEnum.START
344
+ if cmd[0] not in self.keywords.keys() and cmd[0] != '':
345
+ if not word_before_cursor == '':
346
+ suggestions = list(self.keywords.keys())
347
+ elif cmd[0] == '':
348
+ suggestions = list(self.keywords.keys())
349
+ elif isinstance(self.keywords[cmd[0]], list):
350
+ suggestions = self.keywords[cmd[0]]
351
+ elif isinstance(self.keywords[cmd[0]], dict):
352
+ suggestions = list(self.keywords[cmd[0]].keys())
353
+ elif len(cmd) >= 2:
354
+ self.thread_state = StateEnum.START
355
+ if cmd[0] not in self.keywords.keys():
356
+ suggestions = list()
357
+ elif isinstance(self.keywords[cmd[0]], list):
358
+ suggestions = self.keywords[cmd[0]]
359
+ elif isinstance(self.keywords[cmd[0]], dict):
360
+ if cmd[1] in self.keywords[cmd[0]].keys():
361
+ suggestions = self.keywords[cmd[0]][cmd[1]]
362
+ else:
363
+ suggestions = list(self.keywords[cmd[0]].keys())
364
+
365
+ matches = fuzzyfinder(word_before_cursor, suggestions)
366
+ for match in matches:
367
+ yield Completion(match, start_position=-len(word_before_cursor))