dtlpy 1.115.44__py3-none-any.whl → 1.116.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +145 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +959 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +963 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1257 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1504 -1504
  167. dtlpy/repositories/downloader.py +976 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +226 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +419 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1785 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.116.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
@@ -1,367 +1,367 @@
1
- from prompt_toolkit.completion import Completer, Completion
2
- from prompt_toolkit.history import History
3
- from fuzzyfinder.main import fuzzyfinder
4
- import csv
5
- import threading
6
- import os
7
- import datetime
8
- import logging
9
-
10
-
11
- class StateEnum:
12
- """
13
- State enum
14
- """
15
- START = 0
16
- RUNNING = 1
17
- DONE = 2
18
- CONTINUE = 3
19
-
20
-
21
- def get_parser_tree(parser):
22
- """
23
- Creates parser tree for autocomplete
24
- :param parser: parser
25
- :return: parser tree
26
- """
27
- if parser._subparsers is None:
28
- p_keywords = list()
29
- for param in parser._option_string_actions:
30
- if param.startswith('--'):
31
- p_keywords.append(param)
32
- else:
33
- p_keywords = dict()
34
- subparsers = parser._subparsers._group_actions[0].choices
35
- for sub_parser in subparsers:
36
- p_keywords[sub_parser] = get_parser_tree(subparsers[sub_parser])
37
- if 'shell' in p_keywords:
38
- # noinspection PyTypeChecker
39
- p_keywords.pop('shell')
40
- return p_keywords
41
-
42
-
43
- class FileHistory(History):
44
- """
45
- :class:`.History` class that stores all strings in a file.
46
- """
47
-
48
- def __init__(self, filename):
49
- self.filename = filename
50
- super(FileHistory, self).__init__()
51
- self.to_hide = ['password', 'secret']
52
-
53
- def load_history_strings(self):
54
- strings = []
55
- lines = []
56
-
57
- def add():
58
- if lines:
59
- # Join and drop trailing newline.
60
- string = ''.join(lines)[:-1]
61
- hide = any(field in string for field in self.to_hide)
62
- if not hide:
63
- strings.append(string)
64
-
65
- if os.path.exists(self.filename):
66
- with open(self.filename, 'rb') as f:
67
- for line in f:
68
- line = line.decode('utf-8')
69
-
70
- if line.startswith('+'):
71
- lines.append(line[1:])
72
- else:
73
- add()
74
- lines = []
75
-
76
- add()
77
-
78
- # Reverse the order, because newest items have to go first.
79
- return reversed(strings)
80
-
81
- def store_string(self, string):
82
- # Save to file.
83
- with open(self.filename, 'ab') as f:
84
- def write(t):
85
- f.write(t.encode('utf-8'))
86
-
87
- write('\n# %s\n' % datetime.datetime.now(datetime.timezone.utc))
88
- for line in string.split('\n'):
89
- hide = any(field in line for field in self.to_hide)
90
- if not hide:
91
- write('+%s\n' % line)
92
-
93
-
94
- class DlpCompleter(Completer):
95
- """
96
- Autocomplete for dlp shell
97
- """
98
-
99
- def __init__(self, keywords, dlp):
100
- super(DlpCompleter, self).__init__()
101
- # globals
102
- self.keywords = keywords
103
- self.param_suggestions = list()
104
- self.thread_state = StateEnum.START
105
- self.dlp = dlp
106
-
107
- def get_param_suggestions(self, param, word_before_cursor, cmd):
108
- """
109
- Return parap suggestions
110
- :param param:
111
- :param word_before_cursor:
112
- :param cmd:
113
- :return:
114
- """
115
- prev_state = logging.root.manager.disable
116
- try:
117
- logging.root.manager.disable = logging.ERROR
118
- if self.thread_state in [StateEnum.RUNNING, StateEnum.DONE]:
119
- return
120
- else:
121
- if param == '--project-name':
122
- self.thread_state = StateEnum.RUNNING
123
- project_list = self.dlp.projects.list()
124
- self.param_suggestions = ['"{}'.format(project.name) for project in project_list]
125
- self.thread_state = StateEnum.DONE
126
-
127
- elif param == '--package-name':
128
- self.thread_state = StateEnum.RUNNING
129
- if '--project-name' in cmd:
130
- project = self.dlp.projects.get(
131
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
132
- packages = project.packages
133
- else:
134
- packages = self.dlp.packages
135
- package_list = packages.list()
136
- self.param_suggestions = ['"{}'.format(package.name) for package in package_list.items]
137
- self.thread_state = StateEnum.DONE
138
-
139
- elif param == '--service-name':
140
- self.thread_state = StateEnum.RUNNING
141
- if '--package-name' in cmd:
142
- package = self.dlp.packages.get(
143
- package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
144
- services = package.services
145
- elif '--project-name' in cmd:
146
- project = self.dlp.projects.get(
147
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
148
- services = project.services
149
- else:
150
- services = self.dlp.services
151
-
152
- service_list = services.list()
153
- self.param_suggestions = ['"{}'.format(service.name) for service in service_list.items]
154
- self.thread_state = StateEnum.DONE
155
-
156
- elif param == '--trigger-name':
157
- self.thread_state = StateEnum.RUNNING
158
- if '--service-name' in cmd:
159
- service = self.dlp.services.get(
160
- service_name=cmd[cmd.index('--service-name') + 1].replace('"', ''))
161
- triggers = service.triggers
162
- elif '--package-name' in cmd:
163
- package = self.dlp.packages.get(
164
- package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
165
- triggers = package.services
166
- elif '--project-name' in cmd:
167
- project = self.dlp.projects.get(
168
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
169
- triggers = project.services
170
- else:
171
- triggers = self.dlp.services
172
-
173
- trigger_list = triggers.list()
174
- self.param_suggestions = ['"{}'.format(trigger.name) for trigger in trigger_list.items]
175
- self.thread_state = StateEnum.DONE
176
-
177
- elif param == '--dataset-name':
178
- self.thread_state = StateEnum.RUNNING
179
- if '--project-name' in cmd:
180
- project = self.dlp.projects.get(
181
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
182
- dataset_list = project.datasets.list()
183
- else:
184
- project = self.dlp.projects.get()
185
- dataset_list = project.datasets.list()
186
- self.param_suggestions = ['"{}'.format(dataset.name) for dataset in dataset_list]
187
- self.thread_state = StateEnum.DONE
188
-
189
- elif param == '--remote-path':
190
- self.thread_state = StateEnum.RUNNING
191
- if '--project-name' in cmd:
192
- project = self.dlp.projects.get(
193
- project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
194
- else:
195
- project = self.dlp.projects.get()
196
- if '--dataset-name' in cmd:
197
- dataset = project.datasets.get(
198
- dataset_name=cmd[cmd.index('--dataset-name') + 1].replace('"', ''))
199
- else:
200
- dataset = self.dlp.datasets.get()
201
- self.param_suggestions = dataset.directory_tree.dir_names
202
- with_quotation = ['"{}'.format(directory) for directory in dataset.directory_tree.dir_names]
203
- self.param_suggestions += with_quotation
204
- self.thread_state = StateEnum.DONE
205
-
206
- elif param in ['--local-path', '--local-annotations-path', '--service-file']:
207
- self.thread_state = StateEnum.CONTINUE
208
- param = word_before_cursor.replace('"', '')
209
- if param == '':
210
- param, path = os.path.splitdrive(os.getcwd())
211
- param += os.path.sep
212
- self.param_suggestions = ['"{}'.format(os.path.join(param, directory))
213
- for directory in os.listdir(param)
214
- if not directory.startswith('.')]
215
- elif os.path.isdir(os.path.dirname(param)):
216
- base_dir = os.path.dirname(param)
217
- self.param_suggestions = ['"{}'.format(os.path.join(base_dir, directory))
218
- for directory in os.listdir(base_dir)
219
- if (not directory.startswith('.') and
220
- param in '"{}'.format(os.path.join(base_dir, directory)))]
221
-
222
- elif param in ['--annotation-options']:
223
- self.thread_state = StateEnum.CONTINUE
224
- self.param_suggestions = ['mask', 'json', 'instance', '"mask, json"',
225
- '"mask, instance"', '"json, instance"', '"mask, json, instance"']
226
-
227
- elif param in ['--annotation-filter-type']:
228
- self.thread_state = StateEnum.CONTINUE
229
- self.param_suggestions = ['box', 'segment', 'binary', '"box, segment"']
230
-
231
- elif param in ['--annotation-filter-label']:
232
- self.thread_state = StateEnum.CONTINUE
233
-
234
- elif param in ['--actions']:
235
- self.thread_state = StateEnum.CONTINUE
236
- self.param_suggestions = ['Created', 'Updated', 'Deleted', '"Created, Updated"',
237
- '"Created, Deleted"', '"Updated, Deleted"', '"Created, Updated, Deleted"']
238
-
239
- elif param in ['--resource']:
240
- self.thread_state = StateEnum.CONTINUE
241
- self.param_suggestions = [value for key, value in self.dlp.TriggerResource.__dict__.items() if
242
- not key.startswith('_')]
243
-
244
- elif param in ['--package-type']:
245
- self.thread_state = StateEnum.CONTINUE
246
- self.param_suggestions = [value for key, value in self.dlp.PackageCatalog.__dict__.items() if
247
- not key.startswith('_')]
248
-
249
- # elif param in ['cd']:
250
- # self.thread_state = StateEnum.CONTINUE
251
- # if word_before_cursor != '' and os.path.isdir(os.path.join(os.getcwd(), word_before_cursor)):
252
- # self.param_suggestions = [os.path.join(word_before_cursor, directory)
253
- # for directory in
254
- # os.listdir(os.path.join(os.getcwd(), word_before_cursor))
255
- # if os.path.isdir(
256
- # os.path.join(os.getcwd(), word_before_cursor, directory))]
257
- # else:
258
- # self.param_suggestions = [directory for directory in os.listdir(os.getcwd()) if
259
- # os.path.isdir(directory)]
260
-
261
- else:
262
- self.thread_state = StateEnum.START
263
- self.param_suggestions = list()
264
- except Exception:
265
- self.param_suggestions = list()
266
- self.thread_state = StateEnum.START
267
- finally:
268
- logging.root.manager.disable = prev_state
269
-
270
- def need_param(self, cmd, word_before_cursor):
271
- """
272
- :param cmd:
273
- :param word_before_cursor:
274
- """
275
- need_param = False
276
-
277
- try:
278
- bool_flags_list = ['--overwrite', '--with-text', '--deploy', '--not-items-folder', '--encode',
279
- '--without-binaries']
280
-
281
- if len(cmd) > 2 and cmd[-2].startswith('--') and word_before_cursor != '':
282
- need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
283
- elif cmd[-1].startswith('-') and word_before_cursor == '':
284
- need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
285
- elif cmd[0] in ['cd']:
286
- need_param = True
287
- except Exception:
288
- need_param = False
289
-
290
- return need_param
291
-
292
- @staticmethod
293
- def get_param(cmd, word_before_cursor):
294
- """
295
- :param cmd:
296
- :param word_before_cursor:
297
- """
298
- if word_before_cursor == '' or len(cmd) < 2:
299
- param = cmd[-1]
300
- else:
301
- param = cmd[-2]
302
-
303
- return param
304
-
305
- def get_completions(self, document, complete_event):
306
- """
307
- Get command completion
308
- :param document:
309
- :param complete_event:
310
- :return:
311
- """
312
- # fix input
313
- cmd = next(csv.reader([" ".join(document.text.split())], delimiter=' '))
314
-
315
- # get current word
316
- word_before_cursor = document.get_word_before_cursor(WORD=True)
317
-
318
- # suggest keywords
319
- suggestions = list()
320
- if self.need_param(cmd=cmd, word_before_cursor=word_before_cursor):
321
- param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor)
322
- if self.thread_state in [StateEnum.START, StateEnum.CONTINUE]:
323
- if param in ['--project-name',
324
- '--dataset-name',
325
- '--remote-path',
326
- '--package-name',
327
- '--service-name',
328
- '--trigger-name']:
329
- thread = threading.Thread(target=self.get_param_suggestions,
330
- kwargs={"param": param,
331
- 'word_before_cursor': word_before_cursor,
332
- 'cmd': cmd})
333
- thread.daemon = True
334
- thread.start()
335
-
336
- else:
337
- self.get_param_suggestions(param=param, word_before_cursor=word_before_cursor, cmd=cmd)
338
- if self.thread_state in [StateEnum.DONE, StateEnum.CONTINUE]:
339
- suggestions = self.param_suggestions
340
- elif len(cmd) == 0:
341
- suggestions = list(self.keywords.keys())
342
- elif len(cmd) == 1:
343
- self.thread_state = StateEnum.START
344
- if cmd[0] not in self.keywords.keys() and cmd[0] != '':
345
- if not word_before_cursor == '':
346
- suggestions = list(self.keywords.keys())
347
- elif cmd[0] == '':
348
- suggestions = list(self.keywords.keys())
349
- elif isinstance(self.keywords[cmd[0]], list):
350
- suggestions = self.keywords[cmd[0]]
351
- elif isinstance(self.keywords[cmd[0]], dict):
352
- suggestions = list(self.keywords[cmd[0]].keys())
353
- elif len(cmd) >= 2:
354
- self.thread_state = StateEnum.START
355
- if cmd[0] not in self.keywords.keys():
356
- suggestions = list()
357
- elif isinstance(self.keywords[cmd[0]], list):
358
- suggestions = self.keywords[cmd[0]]
359
- elif isinstance(self.keywords[cmd[0]], dict):
360
- if cmd[1] in self.keywords[cmd[0]].keys():
361
- suggestions = self.keywords[cmd[0]][cmd[1]]
362
- else:
363
- suggestions = list(self.keywords[cmd[0]].keys())
364
-
365
- matches = fuzzyfinder(word_before_cursor, suggestions)
366
- for match in matches:
367
- yield Completion(match, start_position=-len(word_before_cursor))
1
+ from prompt_toolkit.completion import Completer, Completion
2
+ from prompt_toolkit.history import History
3
+ from fuzzyfinder.main import fuzzyfinder
4
+ import csv
5
+ import threading
6
+ import os
7
+ import datetime
8
+ import logging
9
+
10
+
11
+ class StateEnum:
12
+ """
13
+ State enum
14
+ """
15
+ START = 0
16
+ RUNNING = 1
17
+ DONE = 2
18
+ CONTINUE = 3
19
+
20
+
21
+ def get_parser_tree(parser):
22
+ """
23
+ Creates parser tree for autocomplete
24
+ :param parser: parser
25
+ :return: parser tree
26
+ """
27
+ if parser._subparsers is None:
28
+ p_keywords = list()
29
+ for param in parser._option_string_actions:
30
+ if param.startswith('--'):
31
+ p_keywords.append(param)
32
+ else:
33
+ p_keywords = dict()
34
+ subparsers = parser._subparsers._group_actions[0].choices
35
+ for sub_parser in subparsers:
36
+ p_keywords[sub_parser] = get_parser_tree(subparsers[sub_parser])
37
+ if 'shell' in p_keywords:
38
+ # noinspection PyTypeChecker
39
+ p_keywords.pop('shell')
40
+ return p_keywords
41
+
42
+
43
+ class FileHistory(History):
44
+ """
45
+ :class:`.History` class that stores all strings in a file.
46
+ """
47
+
48
+ def __init__(self, filename):
49
+ self.filename = filename
50
+ super(FileHistory, self).__init__()
51
+ self.to_hide = ['password', 'secret']
52
+
53
+ def load_history_strings(self):
54
+ strings = []
55
+ lines = []
56
+
57
+ def add():
58
+ if lines:
59
+ # Join and drop trailing newline.
60
+ string = ''.join(lines)[:-1]
61
+ hide = any(field in string for field in self.to_hide)
62
+ if not hide:
63
+ strings.append(string)
64
+
65
+ if os.path.exists(self.filename):
66
+ with open(self.filename, 'rb') as f:
67
+ for line in f:
68
+ line = line.decode('utf-8')
69
+
70
+ if line.startswith('+'):
71
+ lines.append(line[1:])
72
+ else:
73
+ add()
74
+ lines = []
75
+
76
+ add()
77
+
78
+ # Reverse the order, because newest items have to go first.
79
+ return reversed(strings)
80
+
81
+ def store_string(self, string):
82
+ # Save to file.
83
+ with open(self.filename, 'ab') as f:
84
+ def write(t):
85
+ f.write(t.encode('utf-8'))
86
+
87
+ write('\n# %s\n' % datetime.datetime.now(datetime.timezone.utc))
88
+ for line in string.split('\n'):
89
+ hide = any(field in line for field in self.to_hide)
90
+ if not hide:
91
+ write('+%s\n' % line)
92
+
93
+
94
+ class DlpCompleter(Completer):
95
+ """
96
+ Autocomplete for dlp shell
97
+ """
98
+
99
+ def __init__(self, keywords, dlp):
100
+ super(DlpCompleter, self).__init__()
101
+ # globals
102
+ self.keywords = keywords
103
+ self.param_suggestions = list()
104
+ self.thread_state = StateEnum.START
105
+ self.dlp = dlp
106
+
107
+ def get_param_suggestions(self, param, word_before_cursor, cmd):
108
+ """
109
+ Return parap suggestions
110
+ :param param:
111
+ :param word_before_cursor:
112
+ :param cmd:
113
+ :return:
114
+ """
115
+ prev_state = logging.root.manager.disable
116
+ try:
117
+ logging.root.manager.disable = logging.ERROR
118
+ if self.thread_state in [StateEnum.RUNNING, StateEnum.DONE]:
119
+ return
120
+ else:
121
+ if param == '--project-name':
122
+ self.thread_state = StateEnum.RUNNING
123
+ project_list = self.dlp.projects.list()
124
+ self.param_suggestions = ['"{}'.format(project.name) for project in project_list]
125
+ self.thread_state = StateEnum.DONE
126
+
127
+ elif param == '--package-name':
128
+ self.thread_state = StateEnum.RUNNING
129
+ if '--project-name' in cmd:
130
+ project = self.dlp.projects.get(
131
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
132
+ packages = project.packages
133
+ else:
134
+ packages = self.dlp.packages
135
+ package_list = packages.list()
136
+ self.param_suggestions = ['"{}'.format(package.name) for package in package_list.items]
137
+ self.thread_state = StateEnum.DONE
138
+
139
+ elif param == '--service-name':
140
+ self.thread_state = StateEnum.RUNNING
141
+ if '--package-name' in cmd:
142
+ package = self.dlp.packages.get(
143
+ package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
144
+ services = package.services
145
+ elif '--project-name' in cmd:
146
+ project = self.dlp.projects.get(
147
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
148
+ services = project.services
149
+ else:
150
+ services = self.dlp.services
151
+
152
+ service_list = services.list()
153
+ self.param_suggestions = ['"{}'.format(service.name) for service in service_list.items]
154
+ self.thread_state = StateEnum.DONE
155
+
156
+ elif param == '--trigger-name':
157
+ self.thread_state = StateEnum.RUNNING
158
+ if '--service-name' in cmd:
159
+ service = self.dlp.services.get(
160
+ service_name=cmd[cmd.index('--service-name') + 1].replace('"', ''))
161
+ triggers = service.triggers
162
+ elif '--package-name' in cmd:
163
+ package = self.dlp.packages.get(
164
+ package_name=cmd[cmd.index('--package-name') + 1].replace('"', ''))
165
+ triggers = package.services
166
+ elif '--project-name' in cmd:
167
+ project = self.dlp.projects.get(
168
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
169
+ triggers = project.services
170
+ else:
171
+ triggers = self.dlp.services
172
+
173
+ trigger_list = triggers.list()
174
+ self.param_suggestions = ['"{}'.format(trigger.name) for trigger in trigger_list.items]
175
+ self.thread_state = StateEnum.DONE
176
+
177
+ elif param == '--dataset-name':
178
+ self.thread_state = StateEnum.RUNNING
179
+ if '--project-name' in cmd:
180
+ project = self.dlp.projects.get(
181
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
182
+ dataset_list = project.datasets.list()
183
+ else:
184
+ project = self.dlp.projects.get()
185
+ dataset_list = project.datasets.list()
186
+ self.param_suggestions = ['"{}'.format(dataset.name) for dataset in dataset_list]
187
+ self.thread_state = StateEnum.DONE
188
+
189
+ elif param == '--remote-path':
190
+ self.thread_state = StateEnum.RUNNING
191
+ if '--project-name' in cmd:
192
+ project = self.dlp.projects.get(
193
+ project_name=cmd[cmd.index('--project-name') + 1].replace('"', ''))
194
+ else:
195
+ project = self.dlp.projects.get()
196
+ if '--dataset-name' in cmd:
197
+ dataset = project.datasets.get(
198
+ dataset_name=cmd[cmd.index('--dataset-name') + 1].replace('"', ''))
199
+ else:
200
+ dataset = self.dlp.datasets.get()
201
+ self.param_suggestions = dataset.directory_tree.dir_names
202
+ with_quotation = ['"{}'.format(directory) for directory in dataset.directory_tree.dir_names]
203
+ self.param_suggestions += with_quotation
204
+ self.thread_state = StateEnum.DONE
205
+
206
+ elif param in ['--local-path', '--local-annotations-path', '--service-file']:
207
+ self.thread_state = StateEnum.CONTINUE
208
+ param = word_before_cursor.replace('"', '')
209
+ if param == '':
210
+ param, path = os.path.splitdrive(os.getcwd())
211
+ param += os.path.sep
212
+ self.param_suggestions = ['"{}'.format(os.path.join(param, directory))
213
+ for directory in os.listdir(param)
214
+ if not directory.startswith('.')]
215
+ elif os.path.isdir(os.path.dirname(param)):
216
+ base_dir = os.path.dirname(param)
217
+ self.param_suggestions = ['"{}'.format(os.path.join(base_dir, directory))
218
+ for directory in os.listdir(base_dir)
219
+ if (not directory.startswith('.') and
220
+ param in '"{}'.format(os.path.join(base_dir, directory)))]
221
+
222
+ elif param in ['--annotation-options']:
223
+ self.thread_state = StateEnum.CONTINUE
224
+ self.param_suggestions = ['mask', 'json', 'instance', '"mask, json"',
225
+ '"mask, instance"', '"json, instance"', '"mask, json, instance"']
226
+
227
+ elif param in ['--annotation-filter-type']:
228
+ self.thread_state = StateEnum.CONTINUE
229
+ self.param_suggestions = ['box', 'segment', 'binary', '"box, segment"']
230
+
231
+ elif param in ['--annotation-filter-label']:
232
+ self.thread_state = StateEnum.CONTINUE
233
+
234
+ elif param in ['--actions']:
235
+ self.thread_state = StateEnum.CONTINUE
236
+ self.param_suggestions = ['Created', 'Updated', 'Deleted', '"Created, Updated"',
237
+ '"Created, Deleted"', '"Updated, Deleted"', '"Created, Updated, Deleted"']
238
+
239
+ elif param in ['--resource']:
240
+ self.thread_state = StateEnum.CONTINUE
241
+ self.param_suggestions = [value for key, value in self.dlp.TriggerResource.__dict__.items() if
242
+ not key.startswith('_')]
243
+
244
+ elif param in ['--package-type']:
245
+ self.thread_state = StateEnum.CONTINUE
246
+ self.param_suggestions = [value for key, value in self.dlp.PackageCatalog.__dict__.items() if
247
+ not key.startswith('_')]
248
+
249
+ # elif param in ['cd']:
250
+ # self.thread_state = StateEnum.CONTINUE
251
+ # if word_before_cursor != '' and os.path.isdir(os.path.join(os.getcwd(), word_before_cursor)):
252
+ # self.param_suggestions = [os.path.join(word_before_cursor, directory)
253
+ # for directory in
254
+ # os.listdir(os.path.join(os.getcwd(), word_before_cursor))
255
+ # if os.path.isdir(
256
+ # os.path.join(os.getcwd(), word_before_cursor, directory))]
257
+ # else:
258
+ # self.param_suggestions = [directory for directory in os.listdir(os.getcwd()) if
259
+ # os.path.isdir(directory)]
260
+
261
+ else:
262
+ self.thread_state = StateEnum.START
263
+ self.param_suggestions = list()
264
+ except Exception:
265
+ self.param_suggestions = list()
266
+ self.thread_state = StateEnum.START
267
+ finally:
268
+ logging.root.manager.disable = prev_state
269
+
270
+ def need_param(self, cmd, word_before_cursor):
271
+ """
272
+ :param cmd:
273
+ :param word_before_cursor:
274
+ """
275
+ need_param = False
276
+
277
+ try:
278
+ bool_flags_list = ['--overwrite', '--with-text', '--deploy', '--not-items-folder', '--encode',
279
+ '--without-binaries']
280
+
281
+ if len(cmd) > 2 and cmd[-2].startswith('--') and word_before_cursor != '':
282
+ need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
283
+ elif cmd[-1].startswith('-') and word_before_cursor == '':
284
+ need_param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor) not in bool_flags_list
285
+ elif cmd[0] in ['cd']:
286
+ need_param = True
287
+ except Exception:
288
+ need_param = False
289
+
290
+ return need_param
291
+
292
+ @staticmethod
293
+ def get_param(cmd, word_before_cursor):
294
+ """
295
+ :param cmd:
296
+ :param word_before_cursor:
297
+ """
298
+ if word_before_cursor == '' or len(cmd) < 2:
299
+ param = cmd[-1]
300
+ else:
301
+ param = cmd[-2]
302
+
303
+ return param
304
+
305
+ def get_completions(self, document, complete_event):
306
+ """
307
+ Get command completion
308
+ :param document:
309
+ :param complete_event:
310
+ :return:
311
+ """
312
+ # fix input
313
+ cmd = next(csv.reader([" ".join(document.text.split())], delimiter=' '))
314
+
315
+ # get current word
316
+ word_before_cursor = document.get_word_before_cursor(WORD=True)
317
+
318
+ # suggest keywords
319
+ suggestions = list()
320
+ if self.need_param(cmd=cmd, word_before_cursor=word_before_cursor):
321
+ param = self.get_param(cmd=cmd, word_before_cursor=word_before_cursor)
322
+ if self.thread_state in [StateEnum.START, StateEnum.CONTINUE]:
323
+ if param in ['--project-name',
324
+ '--dataset-name',
325
+ '--remote-path',
326
+ '--package-name',
327
+ '--service-name',
328
+ '--trigger-name']:
329
+ thread = threading.Thread(target=self.get_param_suggestions,
330
+ kwargs={"param": param,
331
+ 'word_before_cursor': word_before_cursor,
332
+ 'cmd': cmd})
333
+ thread.daemon = True
334
+ thread.start()
335
+
336
+ else:
337
+ self.get_param_suggestions(param=param, word_before_cursor=word_before_cursor, cmd=cmd)
338
+ if self.thread_state in [StateEnum.DONE, StateEnum.CONTINUE]:
339
+ suggestions = self.param_suggestions
340
+ elif len(cmd) == 0:
341
+ suggestions = list(self.keywords.keys())
342
+ elif len(cmd) == 1:
343
+ self.thread_state = StateEnum.START
344
+ if cmd[0] not in self.keywords.keys() and cmd[0] != '':
345
+ if not word_before_cursor == '':
346
+ suggestions = list(self.keywords.keys())
347
+ elif cmd[0] == '':
348
+ suggestions = list(self.keywords.keys())
349
+ elif isinstance(self.keywords[cmd[0]], list):
350
+ suggestions = self.keywords[cmd[0]]
351
+ elif isinstance(self.keywords[cmd[0]], dict):
352
+ suggestions = list(self.keywords[cmd[0]].keys())
353
+ elif len(cmd) >= 2:
354
+ self.thread_state = StateEnum.START
355
+ if cmd[0] not in self.keywords.keys():
356
+ suggestions = list()
357
+ elif isinstance(self.keywords[cmd[0]], list):
358
+ suggestions = self.keywords[cmd[0]]
359
+ elif isinstance(self.keywords[cmd[0]], dict):
360
+ if cmd[1] in self.keywords[cmd[0]].keys():
361
+ suggestions = self.keywords[cmd[0]][cmd[1]]
362
+ else:
363
+ suggestions = list(self.keywords[cmd[0]].keys())
364
+
365
+ matches = fuzzyfinder(word_before_cursor, suggestions)
366
+ for match in matches:
367
+ yield Completion(match, start_position=-len(word_before_cursor))