dtlpy 1.114.17__py3-none-any.whl → 1.116.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__init__.py +491 -491
- dtlpy/__version__.py +1 -1
- dtlpy/assets/__init__.py +26 -26
- dtlpy/assets/code_server/config.yaml +2 -2
- dtlpy/assets/code_server/installation.sh +24 -24
- dtlpy/assets/code_server/launch.json +13 -13
- dtlpy/assets/code_server/settings.json +2 -2
- dtlpy/assets/main.py +53 -53
- dtlpy/assets/main_partial.py +18 -18
- dtlpy/assets/mock.json +11 -11
- dtlpy/assets/model_adapter.py +83 -83
- dtlpy/assets/package.json +61 -61
- dtlpy/assets/package_catalog.json +29 -29
- dtlpy/assets/package_gitignore +307 -307
- dtlpy/assets/service_runners/__init__.py +33 -33
- dtlpy/assets/service_runners/converter.py +96 -96
- dtlpy/assets/service_runners/multi_method.py +49 -49
- dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
- dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
- dtlpy/assets/service_runners/multi_method_item.py +52 -52
- dtlpy/assets/service_runners/multi_method_json.py +52 -52
- dtlpy/assets/service_runners/single_method.py +37 -37
- dtlpy/assets/service_runners/single_method_annotation.py +43 -43
- dtlpy/assets/service_runners/single_method_dataset.py +43 -43
- dtlpy/assets/service_runners/single_method_item.py +41 -41
- dtlpy/assets/service_runners/single_method_json.py +42 -42
- dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
- dtlpy/assets/voc_annotation_template.xml +23 -23
- dtlpy/caches/base_cache.py +32 -32
- dtlpy/caches/cache.py +473 -473
- dtlpy/caches/dl_cache.py +201 -201
- dtlpy/caches/filesystem_cache.py +89 -89
- dtlpy/caches/redis_cache.py +84 -84
- dtlpy/dlp/__init__.py +20 -20
- dtlpy/dlp/cli_utilities.py +367 -367
- dtlpy/dlp/command_executor.py +764 -764
- dtlpy/dlp/dlp +1 -1
- dtlpy/dlp/dlp.bat +1 -1
- dtlpy/dlp/dlp.py +128 -128
- dtlpy/dlp/parser.py +651 -651
- dtlpy/entities/__init__.py +83 -83
- dtlpy/entities/analytic.py +347 -311
- dtlpy/entities/annotation.py +1879 -1879
- dtlpy/entities/annotation_collection.py +699 -699
- dtlpy/entities/annotation_definitions/__init__.py +20 -20
- dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
- dtlpy/entities/annotation_definitions/box.py +195 -195
- dtlpy/entities/annotation_definitions/classification.py +67 -67
- dtlpy/entities/annotation_definitions/comparison.py +72 -72
- dtlpy/entities/annotation_definitions/cube.py +204 -204
- dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
- dtlpy/entities/annotation_definitions/description.py +32 -32
- dtlpy/entities/annotation_definitions/ellipse.py +124 -124
- dtlpy/entities/annotation_definitions/free_text.py +62 -62
- dtlpy/entities/annotation_definitions/gis.py +69 -69
- dtlpy/entities/annotation_definitions/note.py +139 -139
- dtlpy/entities/annotation_definitions/point.py +117 -117
- dtlpy/entities/annotation_definitions/polygon.py +182 -182
- dtlpy/entities/annotation_definitions/polyline.py +111 -111
- dtlpy/entities/annotation_definitions/pose.py +92 -92
- dtlpy/entities/annotation_definitions/ref_image.py +86 -86
- dtlpy/entities/annotation_definitions/segmentation.py +240 -240
- dtlpy/entities/annotation_definitions/subtitle.py +34 -34
- dtlpy/entities/annotation_definitions/text.py +85 -85
- dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
- dtlpy/entities/app.py +220 -220
- dtlpy/entities/app_module.py +107 -107
- dtlpy/entities/artifact.py +174 -174
- dtlpy/entities/assignment.py +399 -399
- dtlpy/entities/base_entity.py +214 -214
- dtlpy/entities/bot.py +113 -113
- dtlpy/entities/codebase.py +292 -296
- dtlpy/entities/collection.py +38 -38
- dtlpy/entities/command.py +169 -169
- dtlpy/entities/compute.py +449 -442
- dtlpy/entities/dataset.py +1299 -1285
- dtlpy/entities/directory_tree.py +44 -44
- dtlpy/entities/dpk.py +470 -470
- dtlpy/entities/driver.py +235 -223
- dtlpy/entities/execution.py +397 -397
- dtlpy/entities/feature.py +124 -124
- dtlpy/entities/feature_set.py +145 -145
- dtlpy/entities/filters.py +798 -645
- dtlpy/entities/gis_item.py +107 -107
- dtlpy/entities/integration.py +184 -184
- dtlpy/entities/item.py +959 -953
- dtlpy/entities/label.py +123 -123
- dtlpy/entities/links.py +85 -85
- dtlpy/entities/message.py +175 -175
- dtlpy/entities/model.py +684 -684
- dtlpy/entities/node.py +1005 -1005
- dtlpy/entities/ontology.py +810 -803
- dtlpy/entities/organization.py +287 -287
- dtlpy/entities/package.py +657 -657
- dtlpy/entities/package_defaults.py +5 -5
- dtlpy/entities/package_function.py +185 -185
- dtlpy/entities/package_module.py +113 -113
- dtlpy/entities/package_slot.py +118 -118
- dtlpy/entities/paged_entities.py +299 -299
- dtlpy/entities/pipeline.py +624 -624
- dtlpy/entities/pipeline_execution.py +279 -279
- dtlpy/entities/project.py +394 -394
- dtlpy/entities/prompt_item.py +505 -499
- dtlpy/entities/recipe.py +301 -301
- dtlpy/entities/reflect_dict.py +102 -102
- dtlpy/entities/resource_execution.py +138 -138
- dtlpy/entities/service.py +963 -958
- dtlpy/entities/service_driver.py +117 -117
- dtlpy/entities/setting.py +294 -294
- dtlpy/entities/task.py +495 -495
- dtlpy/entities/time_series.py +143 -143
- dtlpy/entities/trigger.py +426 -426
- dtlpy/entities/user.py +118 -118
- dtlpy/entities/webhook.py +124 -124
- dtlpy/examples/__init__.py +19 -19
- dtlpy/examples/add_labels.py +135 -135
- dtlpy/examples/add_metadata_to_item.py +21 -21
- dtlpy/examples/annotate_items_using_model.py +65 -65
- dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
- dtlpy/examples/annotations_convert_to_voc.py +9 -9
- dtlpy/examples/annotations_convert_to_yolo.py +9 -9
- dtlpy/examples/convert_annotation_types.py +51 -51
- dtlpy/examples/converter.py +143 -143
- dtlpy/examples/copy_annotations.py +22 -22
- dtlpy/examples/copy_folder.py +31 -31
- dtlpy/examples/create_annotations.py +51 -51
- dtlpy/examples/create_video_annotations.py +83 -83
- dtlpy/examples/delete_annotations.py +26 -26
- dtlpy/examples/filters.py +113 -113
- dtlpy/examples/move_item.py +23 -23
- dtlpy/examples/play_video_annotation.py +13 -13
- dtlpy/examples/show_item_and_mask.py +53 -53
- dtlpy/examples/triggers.py +49 -49
- dtlpy/examples/upload_batch_of_items.py +20 -20
- dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
- dtlpy/examples/upload_items_with_modalities.py +43 -43
- dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
- dtlpy/examples/upload_yolo_format_annotations.py +70 -70
- dtlpy/exceptions.py +125 -125
- dtlpy/miscellaneous/__init__.py +20 -20
- dtlpy/miscellaneous/dict_differ.py +95 -95
- dtlpy/miscellaneous/git_utils.py +217 -217
- dtlpy/miscellaneous/json_utils.py +14 -14
- dtlpy/miscellaneous/list_print.py +105 -105
- dtlpy/miscellaneous/zipping.py +130 -130
- dtlpy/ml/__init__.py +20 -20
- dtlpy/ml/base_feature_extractor_adapter.py +27 -27
- dtlpy/ml/base_model_adapter.py +1257 -1086
- dtlpy/ml/metrics.py +461 -461
- dtlpy/ml/predictions_utils.py +274 -274
- dtlpy/ml/summary_writer.py +57 -57
- dtlpy/ml/train_utils.py +60 -60
- dtlpy/new_instance.py +252 -252
- dtlpy/repositories/__init__.py +56 -56
- dtlpy/repositories/analytics.py +85 -85
- dtlpy/repositories/annotations.py +916 -916
- dtlpy/repositories/apps.py +383 -383
- dtlpy/repositories/artifacts.py +452 -452
- dtlpy/repositories/assignments.py +599 -599
- dtlpy/repositories/bots.py +213 -213
- dtlpy/repositories/codebases.py +559 -559
- dtlpy/repositories/collections.py +332 -332
- dtlpy/repositories/commands.py +152 -158
- dtlpy/repositories/compositions.py +61 -61
- dtlpy/repositories/computes.py +439 -435
- dtlpy/repositories/datasets.py +1504 -1291
- dtlpy/repositories/downloader.py +976 -903
- dtlpy/repositories/dpks.py +433 -433
- dtlpy/repositories/drivers.py +482 -470
- dtlpy/repositories/executions.py +815 -817
- dtlpy/repositories/feature_sets.py +226 -226
- dtlpy/repositories/features.py +255 -238
- dtlpy/repositories/integrations.py +484 -484
- dtlpy/repositories/items.py +912 -909
- dtlpy/repositories/messages.py +94 -94
- dtlpy/repositories/models.py +1000 -988
- dtlpy/repositories/nodes.py +80 -80
- dtlpy/repositories/ontologies.py +511 -511
- dtlpy/repositories/organizations.py +525 -525
- dtlpy/repositories/packages.py +1941 -1941
- dtlpy/repositories/pipeline_executions.py +451 -451
- dtlpy/repositories/pipelines.py +640 -640
- dtlpy/repositories/projects.py +539 -539
- dtlpy/repositories/recipes.py +419 -399
- dtlpy/repositories/resource_executions.py +137 -137
- dtlpy/repositories/schema.py +120 -120
- dtlpy/repositories/service_drivers.py +213 -213
- dtlpy/repositories/services.py +1704 -1704
- dtlpy/repositories/settings.py +339 -339
- dtlpy/repositories/tasks.py +1477 -1477
- dtlpy/repositories/times_series.py +278 -278
- dtlpy/repositories/triggers.py +536 -536
- dtlpy/repositories/upload_element.py +257 -257
- dtlpy/repositories/uploader.py +661 -651
- dtlpy/repositories/webhooks.py +249 -249
- dtlpy/services/__init__.py +22 -22
- dtlpy/services/aihttp_retry.py +131 -131
- dtlpy/services/api_client.py +1785 -1782
- dtlpy/services/api_reference.py +40 -40
- dtlpy/services/async_utils.py +133 -133
- dtlpy/services/calls_counter.py +44 -44
- dtlpy/services/check_sdk.py +68 -68
- dtlpy/services/cookie.py +115 -115
- dtlpy/services/create_logger.py +156 -156
- dtlpy/services/events.py +84 -84
- dtlpy/services/logins.py +235 -235
- dtlpy/services/reporter.py +256 -256
- dtlpy/services/service_defaults.py +91 -91
- dtlpy/utilities/__init__.py +20 -20
- dtlpy/utilities/annotations/__init__.py +16 -16
- dtlpy/utilities/annotations/annotation_converters.py +269 -269
- dtlpy/utilities/base_package_runner.py +285 -264
- dtlpy/utilities/converter.py +1650 -1650
- dtlpy/utilities/dataset_generators/__init__.py +1 -1
- dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
- dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
- dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
- dtlpy/utilities/local_development/__init__.py +1 -1
- dtlpy/utilities/local_development/local_session.py +179 -179
- dtlpy/utilities/reports/__init__.py +2 -2
- dtlpy/utilities/reports/figures.py +343 -343
- dtlpy/utilities/reports/report.py +71 -71
- dtlpy/utilities/videos/__init__.py +17 -17
- dtlpy/utilities/videos/video_player.py +598 -598
- dtlpy/utilities/videos/videos.py +470 -470
- {dtlpy-1.114.17.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
- dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
- {dtlpy-1.114.17.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -183
- dtlpy-1.116.6.dist-info/RECORD +239 -0
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
- tests/features/environment.py +551 -551
- dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
- dtlpy-1.114.17.data/scripts/dlp.bat +0 -2
- dtlpy-1.114.17.dist-info/RECORD +0 -240
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
dtlpy/dlp/command_executor.py
CHANGED
|
@@ -1,764 +1,764 @@
|
|
|
1
|
-
import subprocess
|
|
2
|
-
import inquirer
|
|
3
|
-
import logging
|
|
4
|
-
import json
|
|
5
|
-
import os
|
|
6
|
-
import sys
|
|
7
|
-
import jwt
|
|
8
|
-
|
|
9
|
-
from .. import exceptions, entities, repositories, utilities, assets
|
|
10
|
-
|
|
11
|
-
logger = logging.getLogger(name='dtlpy')
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class CommandExecutor:
|
|
15
|
-
|
|
16
|
-
def __init__(self, dl, parser):
|
|
17
|
-
self.dl = dl
|
|
18
|
-
self.parser = parser
|
|
19
|
-
self.utils = Utils(dl)
|
|
20
|
-
|
|
21
|
-
def run(self, args):
|
|
22
|
-
########################
|
|
23
|
-
# Run Command if Exist #
|
|
24
|
-
########################
|
|
25
|
-
if args.operation is None:
|
|
26
|
-
logger.info('See "dlp --help" for options')
|
|
27
|
-
return
|
|
28
|
-
|
|
29
|
-
operation = args.operation.lower().replace('-', '_')
|
|
30
|
-
if hasattr(self, operation):
|
|
31
|
-
getattr(self, operation)(args)
|
|
32
|
-
###############
|
|
33
|
-
# Catch typos #
|
|
34
|
-
###############
|
|
35
|
-
elif args.operation in ["project", 'dataset', 'item', 'service', 'package', 'video', 'deploy', 'generate']:
|
|
36
|
-
self.typos(args=args)
|
|
37
|
-
#######################
|
|
38
|
-
# Catch other options #
|
|
39
|
-
#######################
|
|
40
|
-
elif args.operation:
|
|
41
|
-
print('dlp: "%s" is not an dlp command' % args.operation)
|
|
42
|
-
print('See "dlp --help" for options')
|
|
43
|
-
else:
|
|
44
|
-
print('See "dlp --help" for options')
|
|
45
|
-
|
|
46
|
-
def help(self, args):
|
|
47
|
-
self.parser.print_help()
|
|
48
|
-
|
|
49
|
-
def logout(self, args):
|
|
50
|
-
self.dl.logout()
|
|
51
|
-
logger.info('logout successful')
|
|
52
|
-
|
|
53
|
-
# noinspection PyUnusedLocal
|
|
54
|
-
def login(self, args):
|
|
55
|
-
self.dl.login()
|
|
56
|
-
self.dl.info(with_token=False)
|
|
57
|
-
|
|
58
|
-
def login_token(self, args):
|
|
59
|
-
self.dl.login_token(args.token)
|
|
60
|
-
self.dl.info(with_token=False)
|
|
61
|
-
|
|
62
|
-
def login_api_key(self, args):
|
|
63
|
-
self.dl.login_api_key(api_key=args.api_key)
|
|
64
|
-
|
|
65
|
-
def login_secret(self, args):
|
|
66
|
-
self.login_m2m(args=args)
|
|
67
|
-
|
|
68
|
-
def login_m2m(self, args):
|
|
69
|
-
self.dl.login_m2m(email=args.email,
|
|
70
|
-
password=args.password,
|
|
71
|
-
client_id=args.client_id,
|
|
72
|
-
client_secret=args.client_secret)
|
|
73
|
-
self.dl.info(with_token=False)
|
|
74
|
-
|
|
75
|
-
def upgrade(self, args):
|
|
76
|
-
url = 'dtlpy'
|
|
77
|
-
if args.url is None:
|
|
78
|
-
try:
|
|
79
|
-
payload = jwt.decode(self.dl.client_api.token, algorithms=['HS256'],
|
|
80
|
-
verify=False, options={'verify_signature': False})
|
|
81
|
-
if 'admin' in payload['https://dataloop.ai/authorization']['roles']:
|
|
82
|
-
url = "https://storage.googleapis.com/dtlpy/dev/dtlpy-latest-py3-none-any.whl"
|
|
83
|
-
except Exception:
|
|
84
|
-
pass
|
|
85
|
-
else:
|
|
86
|
-
url = args.url
|
|
87
|
-
|
|
88
|
-
logger.info("Update DTLPy from {}".format(url))
|
|
89
|
-
logger.info("Installing using pip...")
|
|
90
|
-
cmd = "pip install {} --upgrade".format(url)
|
|
91
|
-
subprocess.Popen(cmd, shell=True)
|
|
92
|
-
sys.exit(0)
|
|
93
|
-
|
|
94
|
-
# noinspection PyUnusedLocal
|
|
95
|
-
def init(self, args):
|
|
96
|
-
self.dl.init()
|
|
97
|
-
|
|
98
|
-
# noinspection PyUnusedLocal
|
|
99
|
-
def checkout_state(self, args):
|
|
100
|
-
state = self.dl.checkout_state()
|
|
101
|
-
logger.info('Checked-out:')
|
|
102
|
-
for key, val in state.items():
|
|
103
|
-
try:
|
|
104
|
-
msg = '{entity} name: {name}\t{entity} id: {id}'.format(entity=key, name=val['name'], id=val['id'])
|
|
105
|
-
except KeyError:
|
|
106
|
-
msg = '{entity} Not Found'.format(entity=key)
|
|
107
|
-
logger.info(msg)
|
|
108
|
-
|
|
109
|
-
# noinspection PyUnusedLocal
|
|
110
|
-
def version(self, args):
|
|
111
|
-
logger.info("Dataloop SDK Version: {}".format(self.dl.__version__))
|
|
112
|
-
|
|
113
|
-
def development(self, args):
|
|
114
|
-
if args.development == "local":
|
|
115
|
-
# start the local development
|
|
116
|
-
if args.local == 'start':
|
|
117
|
-
development = self.utils.dl.client_api.state_io.get('development')
|
|
118
|
-
# create default values
|
|
119
|
-
if development is None:
|
|
120
|
-
development = dict()
|
|
121
|
-
if 'port' not in development:
|
|
122
|
-
development['port'] = 5802
|
|
123
|
-
if 'docker_image' not in development:
|
|
124
|
-
development['docker_image'] = 'dataloopai/dtlpy-agent:1.57.3.gpu.cuda11.5.py3.8.opencv'
|
|
125
|
-
|
|
126
|
-
# get values from input
|
|
127
|
-
if args.docker_image is not None:
|
|
128
|
-
development['docker_image'] = args.docker_image
|
|
129
|
-
if args.port is not None:
|
|
130
|
-
development['port'] = int(args.port)
|
|
131
|
-
|
|
132
|
-
# set values to local state
|
|
133
|
-
self.utils.dl.client_api.state_io.put('development', development)
|
|
134
|
-
utilities.local_development.start_session()
|
|
135
|
-
elif args.local == 'pause':
|
|
136
|
-
utilities.local_development.pause_session()
|
|
137
|
-
elif args.local == 'stop':
|
|
138
|
-
utilities.local_development.stop_session()
|
|
139
|
-
else:
|
|
140
|
-
print('Must select one of "start", "pause", "stop". Type "dlp development start --help" for options')
|
|
141
|
-
elif args.development == "remote":
|
|
142
|
-
logger.warning('FUTURE! This is not supported yet..')
|
|
143
|
-
else:
|
|
144
|
-
print('Type "dlp development --help" for options')
|
|
145
|
-
|
|
146
|
-
# noinspection PyUnusedLocal
|
|
147
|
-
|
|
148
|
-
def api(self, args):
|
|
149
|
-
if args.api == "info":
|
|
150
|
-
information = self.dl.info()
|
|
151
|
-
logger.info('-- Dataloop info --')
|
|
152
|
-
_ = [logger.info('{}: {}'.format(key, val)) for key, val in information.items()]
|
|
153
|
-
|
|
154
|
-
if args.api == "setenv":
|
|
155
|
-
self.dl.setenv(args.env)
|
|
156
|
-
logger.info("Platform environment: {}".format(self.dl.environment()))
|
|
157
|
-
|
|
158
|
-
def projects(self, args):
|
|
159
|
-
if args.projects == "ls":
|
|
160
|
-
self.dl.projects.list().print()
|
|
161
|
-
elif args.projects == "web":
|
|
162
|
-
if args.project_name is None:
|
|
163
|
-
args.project_name = self.dl.projects.get().name
|
|
164
|
-
self.dl.projects.open_in_web(project_name=args.project_name)
|
|
165
|
-
elif args.projects == "create":
|
|
166
|
-
project = self.dl.projects.create(args.project_name)
|
|
167
|
-
project.print()
|
|
168
|
-
project.checkout()
|
|
169
|
-
elif args.projects == "checkout":
|
|
170
|
-
self.dl.projects.checkout(project_name=args.project_name)
|
|
171
|
-
else:
|
|
172
|
-
print('Type "dlp projects --help" for options')
|
|
173
|
-
|
|
174
|
-
def datasets(self, args):
|
|
175
|
-
if args.datasets == "ls":
|
|
176
|
-
self.utils.get_datasets_repo(args=args).list().print()
|
|
177
|
-
|
|
178
|
-
elif args.datasets == "checkout":
|
|
179
|
-
self.dl.datasets.checkout(dataset_name=args.dataset_name)
|
|
180
|
-
|
|
181
|
-
elif args.datasets == "web":
|
|
182
|
-
if args.dataset_name is None:
|
|
183
|
-
args.dataset_name = self.dl.datasets.get().name
|
|
184
|
-
self.utils.get_datasets_repo(args=args).open_in_web(dataset_name=args.dataset_name)
|
|
185
|
-
|
|
186
|
-
elif args.datasets == "create":
|
|
187
|
-
dataset = self.utils.get_datasets_repo(args=args).create(dataset_name=args.dataset_name)
|
|
188
|
-
dataset.print()
|
|
189
|
-
if args.checkout:
|
|
190
|
-
dataset.checkout()
|
|
191
|
-
else:
|
|
192
|
-
print('Type "dlp datasets --help" for options')
|
|
193
|
-
|
|
194
|
-
def items(self, args):
|
|
195
|
-
if self.dl.token_expired():
|
|
196
|
-
logger.error("token expired, please login.")
|
|
197
|
-
return
|
|
198
|
-
|
|
199
|
-
if args.items == "ls":
|
|
200
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
201
|
-
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
202
|
-
if isinstance(args.page, str):
|
|
203
|
-
try:
|
|
204
|
-
args.page = int(args.page)
|
|
205
|
-
except ValueError:
|
|
206
|
-
raise ValueError("Input --page must be integer")
|
|
207
|
-
filters = self.dl.Filters()
|
|
208
|
-
|
|
209
|
-
# add filters
|
|
210
|
-
if args.remote_path is not None:
|
|
211
|
-
if isinstance(args.remote_path, list):
|
|
212
|
-
filters.add(field="filename", values=args.remote_path, operator=entities.FiltersOperations.IN)
|
|
213
|
-
else:
|
|
214
|
-
filters.add(field="filename", values=args.remote_path)
|
|
215
|
-
if args.type is not None:
|
|
216
|
-
if isinstance(args.type, list):
|
|
217
|
-
filters.add(field='metadata.system.mimetype', values=args.type,
|
|
218
|
-
operator=entities.FiltersOperations.IN)
|
|
219
|
-
else:
|
|
220
|
-
filters.add(field="metadata.system.mimetype", values=args.type)
|
|
221
|
-
|
|
222
|
-
pages = dataset.items.list(filters=filters, page_offset=args.page)
|
|
223
|
-
pages.print()
|
|
224
|
-
print("Displaying page %d/%d" % (args.page, pages.total_pages_count - 1))
|
|
225
|
-
|
|
226
|
-
elif args.items == "web":
|
|
227
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
228
|
-
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
229
|
-
dataset.items.open_in_web(filepath=args.remote_path)
|
|
230
|
-
|
|
231
|
-
elif args.items == "upload":
|
|
232
|
-
logger.info("Uploading directory...")
|
|
233
|
-
if isinstance(args.file_types, str):
|
|
234
|
-
args.file_types = [t.strip() for t in args.file_types.split(",")]
|
|
235
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
236
|
-
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
237
|
-
|
|
238
|
-
dataset.items.upload(local_path=args.local_path,
|
|
239
|
-
remote_path=args.remote_path,
|
|
240
|
-
file_types=args.file_types,
|
|
241
|
-
overwrite=args.overwrite,
|
|
242
|
-
local_annotations_path=args.local_annotations_path)
|
|
243
|
-
|
|
244
|
-
elif args.items == "download":
|
|
245
|
-
logger.info("Downloading dataset...")
|
|
246
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
247
|
-
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
248
|
-
|
|
249
|
-
annotation_options = None
|
|
250
|
-
if args.annotation_options is not None:
|
|
251
|
-
annotation_options = [t.strip() for t in args.annotation_options.split(",")]
|
|
252
|
-
|
|
253
|
-
annotation_filters = None
|
|
254
|
-
if args.annotation_filter_type is not None or args.annotation_filter_label is not None:
|
|
255
|
-
annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
|
|
256
|
-
if args.annotation_filter_type is not None:
|
|
257
|
-
annotation_filter_type = [t.strip() for t in args.annotation_filter_type.split(",")]
|
|
258
|
-
annotation_filters.add(field='type',
|
|
259
|
-
values=annotation_filter_type,
|
|
260
|
-
operator=entities.FiltersOperations.IN)
|
|
261
|
-
if args.annotation_filter_label is not None:
|
|
262
|
-
annotation_filter_label = [t.strip() for t in args.annotation_filter_label.split(",")]
|
|
263
|
-
annotation_filters.add(field='label',
|
|
264
|
-
values=annotation_filter_label,
|
|
265
|
-
operator=entities.FiltersOperations.IN)
|
|
266
|
-
# create remote path filters
|
|
267
|
-
filters = self.dl.Filters()
|
|
268
|
-
if args.remote_path is not None:
|
|
269
|
-
remote_path = [t.strip() for t in args.remote_path.split(",")]
|
|
270
|
-
if len(remote_path) == 1:
|
|
271
|
-
remote_path = remote_path[0]
|
|
272
|
-
filters.add(field="filename", values=remote_path)
|
|
273
|
-
elif len(remote_path) > 1:
|
|
274
|
-
for item in remote_path:
|
|
275
|
-
if '*' in item:
|
|
276
|
-
filters.add(field="dir", values=item, method='or')
|
|
277
|
-
remote_path.pop(remote_path.index(item))
|
|
278
|
-
filters.add(field="dir", values=remote_path, operator=entities.FiltersOperations.IN, method='or')
|
|
279
|
-
|
|
280
|
-
if not args.without_binaries:
|
|
281
|
-
dataset.items.download(filters=filters,
|
|
282
|
-
local_path=args.local_path,
|
|
283
|
-
annotation_options=annotation_options,
|
|
284
|
-
annotation_filters=annotation_filters,
|
|
285
|
-
overwrite=args.overwrite,
|
|
286
|
-
with_text=args.with_text,
|
|
287
|
-
thickness=int(args.thickness),
|
|
288
|
-
to_items_folder=not args.not_items_folder)
|
|
289
|
-
else:
|
|
290
|
-
dataset.download_annotations(filters=filters,
|
|
291
|
-
local_path=args.local_path,
|
|
292
|
-
annotation_options=annotation_options,
|
|
293
|
-
annotation_filters=annotation_filters,
|
|
294
|
-
overwrite=args.overwrite,
|
|
295
|
-
with_text=args.with_text,
|
|
296
|
-
thickness=int(args.thickness))
|
|
297
|
-
|
|
298
|
-
else:
|
|
299
|
-
print('Type "dlp items --help" for options')
|
|
300
|
-
|
|
301
|
-
def videos(self, args):
|
|
302
|
-
if self.dl.token_expired():
|
|
303
|
-
print("[ERROR] token expired, please login.")
|
|
304
|
-
return
|
|
305
|
-
|
|
306
|
-
if args.videos == "play":
|
|
307
|
-
from dtlpy.utilities.videos.video_player import VideoPlayer
|
|
308
|
-
|
|
309
|
-
VideoPlayer(
|
|
310
|
-
item_filepath=args.item_path,
|
|
311
|
-
dataset_name=args.dataset_name,
|
|
312
|
-
project_name=args.project_name,
|
|
313
|
-
)
|
|
314
|
-
|
|
315
|
-
elif args.videos == "upload":
|
|
316
|
-
if (
|
|
317
|
-
(args.split_chunks is not None)
|
|
318
|
-
or (args.split_seconds is not None)
|
|
319
|
-
or (args.split_times is not None)
|
|
320
|
-
):
|
|
321
|
-
# upload with split
|
|
322
|
-
if isinstance(args.split_chunks, str):
|
|
323
|
-
args.split_chunks = int(args.split_chunks)
|
|
324
|
-
if isinstance(args.split_seconds, str):
|
|
325
|
-
args.split_seconds = int(args.split_seconds)
|
|
326
|
-
if isinstance(args.split_times, str):
|
|
327
|
-
args.split_times = [int(sec) for sec in args.split_times.split(",")]
|
|
328
|
-
self.dl.utilities.videos.Videos.split_and_upload(
|
|
329
|
-
project_name=args.project_name,
|
|
330
|
-
dataset_name=args.dataset_name,
|
|
331
|
-
filepath=args.filename,
|
|
332
|
-
remote_path=args.remote_path,
|
|
333
|
-
split_chunks=args.split_chunks,
|
|
334
|
-
split_seconds=args.split_seconds,
|
|
335
|
-
split_pairs=args.split_times,
|
|
336
|
-
)
|
|
337
|
-
else:
|
|
338
|
-
print('Type "dlp videos --help" for options')
|
|
339
|
-
|
|
340
|
-
def services(self, args):
|
|
341
|
-
if self.dl.token_expired():
|
|
342
|
-
print("[ERROR] token expired, please login.")
|
|
343
|
-
return
|
|
344
|
-
|
|
345
|
-
elif args.services == "delete":
|
|
346
|
-
service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
|
|
347
|
-
service.delete()
|
|
348
|
-
logger.info('Service: "{}" deleted successfully'.format(service.name))
|
|
349
|
-
|
|
350
|
-
elif args.services == "ls":
|
|
351
|
-
self.utils.get_services_repo(args=args).list().print()
|
|
352
|
-
elif args.services == "log":
|
|
353
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
354
|
-
service = project.services.get(service_name=args.service_name)
|
|
355
|
-
|
|
356
|
-
logs = service.log(start=args.start)
|
|
357
|
-
try:
|
|
358
|
-
for log in logs:
|
|
359
|
-
if isinstance(log, list):
|
|
360
|
-
for log_record in log:
|
|
361
|
-
print(log_record)
|
|
362
|
-
else:
|
|
363
|
-
print(log)
|
|
364
|
-
except KeyboardInterrupt:
|
|
365
|
-
pass
|
|
366
|
-
|
|
367
|
-
elif args.services == "execute":
|
|
368
|
-
try:
|
|
369
|
-
if args.service_name:
|
|
370
|
-
service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
|
|
371
|
-
else:
|
|
372
|
-
service = self.dl.services.get()
|
|
373
|
-
except Exception:
|
|
374
|
-
logger.info('Please checkout a service')
|
|
375
|
-
return
|
|
376
|
-
|
|
377
|
-
if args.annotation_id is not None:
|
|
378
|
-
resource = 'Annotation'
|
|
379
|
-
elif args.item_id is not None:
|
|
380
|
-
resource = 'Item'
|
|
381
|
-
elif args.dataset_id is not None:
|
|
382
|
-
resource = 'Dataset'
|
|
383
|
-
else:
|
|
384
|
-
resource = None
|
|
385
|
-
|
|
386
|
-
try:
|
|
387
|
-
execution_input = json.loads(args.inputs)
|
|
388
|
-
except Exception:
|
|
389
|
-
logger.info('Input should be json serializable')
|
|
390
|
-
return
|
|
391
|
-
if len(execution_input) == 0:
|
|
392
|
-
execution_input = None
|
|
393
|
-
|
|
394
|
-
logger.info('executing')
|
|
395
|
-
service.execute(sync=not args.asynchronous,
|
|
396
|
-
execution_input=execution_input,
|
|
397
|
-
function_name=args.function_name,
|
|
398
|
-
resource=resource,
|
|
399
|
-
item_id=args.item_id,
|
|
400
|
-
dataset_id=args.dataset_id,
|
|
401
|
-
annotation_id=args.annotation_id)
|
|
402
|
-
logger.info("Successfully executed function: {}".format(args.function_name))
|
|
403
|
-
|
|
404
|
-
else:
|
|
405
|
-
logger.info('Type "dlp packages --help" for options')
|
|
406
|
-
|
|
407
|
-
def deploy(self, args):
|
|
408
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
409
|
-
json_filepath = args.json_file
|
|
410
|
-
deployed_services, package = self.dl.packages.deploy_from_file(project=project, json_filepath=json_filepath)
|
|
411
|
-
logger.info("Successfully deployed {} from file: {}\nServices: {}".format(len(deployed_services),
|
|
412
|
-
json_filepath,
|
|
413
|
-
[s.name for s in deployed_services]))
|
|
414
|
-
|
|
415
|
-
def generate(self, args):
|
|
416
|
-
package_type = args.package_type if args.package_type else self.dl.PackageCatalog.DEFAULT_PACKAGE_TYPE
|
|
417
|
-
self.dl.packages.generate(name=args.package_name, src_path=os.getcwd(), package_type=package_type)
|
|
418
|
-
self.utils.dl.client_api.state_io.put('package', {'name': args.package_name})
|
|
419
|
-
logger.info('Successfully generated package files')
|
|
420
|
-
|
|
421
|
-
def triggers(self, args):
|
|
422
|
-
|
|
423
|
-
if args.triggers == "create":
|
|
424
|
-
args.actions = [t.strip() for t in args.actions.split(",")]
|
|
425
|
-
try:
|
|
426
|
-
service = self.utils.get_services_repo(args).get(service_name=args.service_name)
|
|
427
|
-
except exceptions.NotFound:
|
|
428
|
-
logger.critical('Service not found. Please check-out a service or provide valid service name')
|
|
429
|
-
return
|
|
430
|
-
|
|
431
|
-
trigger = service.triggers.create(name=args.name,
|
|
432
|
-
filters=json.loads('{}'.format(args.filters.replace("'", '"'))),
|
|
433
|
-
function_name=args.function_name,
|
|
434
|
-
resource=args.resource,
|
|
435
|
-
actions=args.actions)
|
|
436
|
-
logger.info('Trigger created successfully: {}'.format(trigger.name))
|
|
437
|
-
|
|
438
|
-
elif args.triggers == "delete":
|
|
439
|
-
triggers = self.utils.get_triggers_repo(args=args)
|
|
440
|
-
triggers.get(trigger_name=args.trigger_name).delete()
|
|
441
|
-
logger.info('Trigger deleted successfully: {}'.format(args.trigger_name))
|
|
442
|
-
elif args.triggers == "ls":
|
|
443
|
-
triggers = self.utils.get_triggers_repo(args=args)
|
|
444
|
-
triggers.list().print()
|
|
445
|
-
else:
|
|
446
|
-
logger.info('Type "dlp packages --help" for options')
|
|
447
|
-
|
|
448
|
-
def packages(self, args):
|
|
449
|
-
if self.dl.token_expired():
|
|
450
|
-
logger.error("token expired, please login.")
|
|
451
|
-
return
|
|
452
|
-
|
|
453
|
-
elif args.packages == "delete":
|
|
454
|
-
package = self.utils.get_packages_repo(args=args).get(package_name=args.package_name)
|
|
455
|
-
package.delete()
|
|
456
|
-
logger.info('Successfully deleted package {}'.format(package.name))
|
|
457
|
-
|
|
458
|
-
elif args.packages == "ls":
|
|
459
|
-
self.utils.get_packages_repo(args=args).list().print()
|
|
460
|
-
|
|
461
|
-
elif args.packages == "checkout":
|
|
462
|
-
self.dl.packages.checkout(package_name=args.package_name)
|
|
463
|
-
|
|
464
|
-
elif args.packages == "push":
|
|
465
|
-
packages = self.utils.get_packages_repo(args=args)
|
|
466
|
-
|
|
467
|
-
package = packages.push(src_path=args.src_path,
|
|
468
|
-
package_name=args.package_name,
|
|
469
|
-
checkout=args.checkout)
|
|
470
|
-
|
|
471
|
-
logger.info("Successfully pushed package to platform\n"
|
|
472
|
-
"Package id:{}\nPackage version:{}".format(package.id,
|
|
473
|
-
package.version))
|
|
474
|
-
elif args.packages == "deploy":
|
|
475
|
-
packages = self.utils.get_packages_repo(args=args)
|
|
476
|
-
|
|
477
|
-
package = packages.deploy(package_name=args.package_name,
|
|
478
|
-
checkout=args.checkout,
|
|
479
|
-
module_name=args.module_name)
|
|
480
|
-
|
|
481
|
-
logger.info("Successfully pushed package to platform\n"
|
|
482
|
-
"Package id:{}\nPackage version:{}".format(package.id,
|
|
483
|
-
package.version))
|
|
484
|
-
|
|
485
|
-
elif args.packages == "test":
|
|
486
|
-
go_back = False
|
|
487
|
-
if 'src' in os.listdir(os.getcwd()):
|
|
488
|
-
go_back = True
|
|
489
|
-
os.chdir('./src')
|
|
490
|
-
try:
|
|
491
|
-
self.dl.packages.test_local_package(concurrency=int(args.concurrency),
|
|
492
|
-
function_name=args.function_name)
|
|
493
|
-
except Exception:
|
|
494
|
-
logger.exception('failed during test')
|
|
495
|
-
finally:
|
|
496
|
-
if go_back:
|
|
497
|
-
os.chdir('..')
|
|
498
|
-
|
|
499
|
-
else:
|
|
500
|
-
logger.info('Type "dlp packages --help" for options')
|
|
501
|
-
|
|
502
|
-
def app(self, args):
|
|
503
|
-
if args.app == 'pack':
|
|
504
|
-
path = self.dl.dpks.pack()
|
|
505
|
-
logger.info(f'Packed to {path}')
|
|
506
|
-
elif args.app == 'init':
|
|
507
|
-
app_filename = assets.paths.APP_JSON_FILENAME
|
|
508
|
-
if os.path.isfile(app_filename):
|
|
509
|
-
questions = [
|
|
510
|
-
inquirer.Confirm(name='overwrite',
|
|
511
|
-
message=f"Dataloop app already initialized. Re-initialize?",
|
|
512
|
-
default=False)]
|
|
513
|
-
answers = inquirer.prompt(questions)
|
|
514
|
-
if answers.get('overwrite') is False:
|
|
515
|
-
return
|
|
516
|
-
name = args.name
|
|
517
|
-
description = args.description
|
|
518
|
-
attributes = args.attributes
|
|
519
|
-
icon = args.icon
|
|
520
|
-
scope = args.scope
|
|
521
|
-
as_array = [name, description, attributes, icon, scope]
|
|
522
|
-
if as_array.count(None) == len(as_array): # No one value is initialized
|
|
523
|
-
dir_name = os.path.basename(os.getcwd())
|
|
524
|
-
questions = [
|
|
525
|
-
inquirer.Text(name='name',
|
|
526
|
-
message=f"Enter the name of the app (or press enter for '{dir_name}'):",
|
|
527
|
-
default=dir_name),
|
|
528
|
-
inquirer.Text(name='description',
|
|
529
|
-
message="Enter the description (or enter for empty): "),
|
|
530
|
-
inquirer.Text(name='attributes',
|
|
531
|
-
message="Enter the attributes (an object, or enter for empty): ",
|
|
532
|
-
default=None),
|
|
533
|
-
inquirer.Text(name='icon',
|
|
534
|
-
message="Enter the path to the icon (or enter for empty): "),
|
|
535
|
-
inquirer.Text(name='scope',
|
|
536
|
-
message="Enter the scope (or enter for 'organization'): ",
|
|
537
|
-
default='project'),
|
|
538
|
-
]
|
|
539
|
-
answers = inquirer.prompt(questions)
|
|
540
|
-
name = answers.get('name')
|
|
541
|
-
description = answers.get('description')
|
|
542
|
-
attributes = answers.get('attributes')
|
|
543
|
-
icon = answers.get('icon')
|
|
544
|
-
scope = answers.get('scope')
|
|
545
|
-
|
|
546
|
-
if attributes is not None:
|
|
547
|
-
attributes = json.loads(attributes)
|
|
548
|
-
|
|
549
|
-
self.dl.dpks.init(name=name,
|
|
550
|
-
description=description,
|
|
551
|
-
attributes=attributes,
|
|
552
|
-
icon=icon,
|
|
553
|
-
scope=scope)
|
|
554
|
-
elif args.app == 'add':
|
|
555
|
-
if args.panel is True:
|
|
556
|
-
default_panel_name = "myPanel"
|
|
557
|
-
choices = list(entities.dpk.SlotType)
|
|
558
|
-
|
|
559
|
-
questions = [
|
|
560
|
-
inquirer.Text(name='name',
|
|
561
|
-
message=f"Enter PANEL NAME (or press enter for '{default_panel_name}'): ",
|
|
562
|
-
default='default_panel_name'),
|
|
563
|
-
inquirer.List(name='support_slot_type',
|
|
564
|
-
message="Enter SUPPORTED SLOT TYPE:",
|
|
565
|
-
choices=choices),
|
|
566
|
-
]
|
|
567
|
-
|
|
568
|
-
answers = inquirer.prompt(questions)
|
|
569
|
-
#####
|
|
570
|
-
# create a dir for that panel
|
|
571
|
-
os.makedirs(answers.get('name'), exist_ok=True)
|
|
572
|
-
# dump to dataloop.json
|
|
573
|
-
app_filename = assets.paths.APP_JSON_FILENAME
|
|
574
|
-
if not os.path.isfile(app_filename):
|
|
575
|
-
logger.error(f"Can't find app config file ({app_filename}), please run `dlp app init` first")
|
|
576
|
-
else:
|
|
577
|
-
with open(app_filename, 'r') as f:
|
|
578
|
-
dpk = entities.Dpk.from_json(json.load(f))
|
|
579
|
-
dpk.components.panels.append(entities.Panel(name=answers.get('name'),
|
|
580
|
-
supported_slots=[answers.get('support_slot_type')]))
|
|
581
|
-
with open(app_filename, 'w') as f:
|
|
582
|
-
json.dump(dpk.to_json(), f, indent=4)
|
|
583
|
-
|
|
584
|
-
elif args.app == 'publish':
|
|
585
|
-
dpk = self.utils.get_dpks_repo(args).publish()
|
|
586
|
-
if dpk:
|
|
587
|
-
logger.info(f'Published the application: id={dpk.id}')
|
|
588
|
-
else:
|
|
589
|
-
logger.info("Couldn't publish the application")
|
|
590
|
-
elif args.app == 'update':
|
|
591
|
-
# TODO: I think it changed, not implemented
|
|
592
|
-
logger.info('App updated successfully')
|
|
593
|
-
elif args.app == 'resume':
|
|
594
|
-
succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
|
|
595
|
-
if succeed is True:
|
|
596
|
-
logger.info('Resumed application successfully')
|
|
597
|
-
else:
|
|
598
|
-
logger.info('Application resume failed')
|
|
599
|
-
elif args.app == 'pause':
|
|
600
|
-
succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
|
|
601
|
-
if succeed is True:
|
|
602
|
-
logger.info('Paused application successfully')
|
|
603
|
-
else:
|
|
604
|
-
logger.info('Application pause failed')
|
|
605
|
-
elif args.app == 'install':
|
|
606
|
-
app = self.utils.get_apps_repo(args).install(
|
|
607
|
-
dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
|
|
608
|
-
organization_id=args.org_id
|
|
609
|
-
)
|
|
610
|
-
if app is not None:
|
|
611
|
-
logger.info('App installed successfully')
|
|
612
|
-
else:
|
|
613
|
-
logger.info("Couldn't install the app")
|
|
614
|
-
elif args.app == 'pull':
|
|
615
|
-
succeed = self.dl.dpks.pull(dpk_name=args.dpk_name)
|
|
616
|
-
if succeed is True:
|
|
617
|
-
logger.info("Pulled successfully")
|
|
618
|
-
else:
|
|
619
|
-
logger.info("Couldn't pull")
|
|
620
|
-
elif args.app == 'list':
|
|
621
|
-
self.dl.apps.list().print()
|
|
622
|
-
|
|
623
|
-
# noinspection PyUnusedLocal
|
|
624
|
-
@staticmethod
|
|
625
|
-
def pwd(args):
|
|
626
|
-
print(os.getcwd())
|
|
627
|
-
|
|
628
|
-
@staticmethod
|
|
629
|
-
def cd(args):
|
|
630
|
-
directory = args.dir
|
|
631
|
-
if directory == '..':
|
|
632
|
-
directory = os.path.split(os.getcwd())[0]
|
|
633
|
-
os.chdir(directory)
|
|
634
|
-
print(os.getcwd())
|
|
635
|
-
|
|
636
|
-
@staticmethod
|
|
637
|
-
def mkdir(args):
|
|
638
|
-
os.mkdir(args.name)
|
|
639
|
-
|
|
640
|
-
# noinspection PyUnusedLocal
|
|
641
|
-
@staticmethod
|
|
642
|
-
def ls(args):
|
|
643
|
-
print(os.getcwd())
|
|
644
|
-
dirs = os.listdir(os.getcwd())
|
|
645
|
-
import pprint
|
|
646
|
-
|
|
647
|
-
pp = pprint.PrettyPrinter(indent=3)
|
|
648
|
-
pp.pprint(dirs)
|
|
649
|
-
|
|
650
|
-
# noinspection PyUnusedLocal
|
|
651
|
-
@staticmethod
|
|
652
|
-
def clear(args):
|
|
653
|
-
if os.name == "nt":
|
|
654
|
-
os.system('cls')
|
|
655
|
-
else:
|
|
656
|
-
os.system('clear')
|
|
657
|
-
|
|
658
|
-
@staticmethod
|
|
659
|
-
def typos(args):
|
|
660
|
-
print('dlp: "{op}" is not an dlp command. Did you mean "{op}s"?'.format(op=args.operation))
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
class Utils:
|
|
664
|
-
def __init__(self, dl):
|
|
665
|
-
self.dl = dl
|
|
666
|
-
|
|
667
|
-
def get_packages_repo(self, args):
|
|
668
|
-
if args.project_name is not None:
|
|
669
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
670
|
-
packages = project.packages
|
|
671
|
-
else:
|
|
672
|
-
try:
|
|
673
|
-
packages = self.dl.projects.get().packages
|
|
674
|
-
except Exception:
|
|
675
|
-
packages = self.dl.packages
|
|
676
|
-
|
|
677
|
-
assert isinstance(packages, repositories.Packages)
|
|
678
|
-
return packages
|
|
679
|
-
|
|
680
|
-
def get_datasets_repo(self, args):
|
|
681
|
-
if args.project_name is not None:
|
|
682
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
683
|
-
datasets = project.datasets
|
|
684
|
-
else:
|
|
685
|
-
try:
|
|
686
|
-
datasets = self.dl.projects.get().datasets
|
|
687
|
-
except Exception:
|
|
688
|
-
datasets = self.dl.datasets
|
|
689
|
-
|
|
690
|
-
assert isinstance(datasets, repositories.Datasets)
|
|
691
|
-
return datasets
|
|
692
|
-
|
|
693
|
-
def get_services_repo(self, args):
|
|
694
|
-
if args.project_name is not None:
|
|
695
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
696
|
-
packages = project.packages
|
|
697
|
-
services = project.services
|
|
698
|
-
else:
|
|
699
|
-
try:
|
|
700
|
-
packages = self.dl.projects.get().packages
|
|
701
|
-
services = self.dl.projects.get().services
|
|
702
|
-
except Exception:
|
|
703
|
-
packages = self.dl.packages
|
|
704
|
-
services = self.dl.services
|
|
705
|
-
|
|
706
|
-
if args.package_name is not None:
|
|
707
|
-
package = packages.get(package_name=args.package_name)
|
|
708
|
-
services = package.services
|
|
709
|
-
|
|
710
|
-
assert isinstance(services, repositories.Services)
|
|
711
|
-
return services
|
|
712
|
-
|
|
713
|
-
def get_apps_repo(self, args) -> repositories.Apps:
|
|
714
|
-
if args.project_name is not None:
|
|
715
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
716
|
-
apps = project.apps
|
|
717
|
-
else:
|
|
718
|
-
try:
|
|
719
|
-
apps = self.dl.projects.get().apps
|
|
720
|
-
except Exception:
|
|
721
|
-
apps = self.dl.apps
|
|
722
|
-
assert isinstance(apps, repositories.Apps)
|
|
723
|
-
return apps
|
|
724
|
-
|
|
725
|
-
def get_dpks_repo(self, args) -> repositories.Dpks:
|
|
726
|
-
if args.project_name is None and args.project_id is None:
|
|
727
|
-
try:
|
|
728
|
-
dpks = self.dl.projects.get().dpks
|
|
729
|
-
except Exception:
|
|
730
|
-
dpks = self.dl.dpks
|
|
731
|
-
else:
|
|
732
|
-
project = self.dl.projects.get(project_name=args.project_name, project_id=args.project_id)
|
|
733
|
-
dpks = project.dpks
|
|
734
|
-
if dpks.project is None:
|
|
735
|
-
raise ValueError("Must input one of `project-name` or `project-id`") from None
|
|
736
|
-
assert isinstance(dpks, repositories.Dpks)
|
|
737
|
-
return dpks
|
|
738
|
-
|
|
739
|
-
def get_triggers_repo(self, args):
|
|
740
|
-
if args.project_name is not None:
|
|
741
|
-
project = self.dl.projects.get(project_name=args.project_name)
|
|
742
|
-
packages = project.packages
|
|
743
|
-
services = project.services
|
|
744
|
-
triggers = project.triggers
|
|
745
|
-
else:
|
|
746
|
-
try:
|
|
747
|
-
packages = self.dl.projects.get().packages
|
|
748
|
-
services = self.dl.projects.get().services
|
|
749
|
-
triggers = self.dl.projects.get().triggers
|
|
750
|
-
except Exception:
|
|
751
|
-
packages = self.dl.packages
|
|
752
|
-
services = self.dl.services
|
|
753
|
-
triggers = self.dl.triggers
|
|
754
|
-
|
|
755
|
-
if args.package_name is not None:
|
|
756
|
-
package = packages.package_name(name=args.package_name)
|
|
757
|
-
services = package.services
|
|
758
|
-
|
|
759
|
-
if args.service_name is not None:
|
|
760
|
-
service = services.get(service_name=args.service_name)
|
|
761
|
-
triggers = service.triggers
|
|
762
|
-
|
|
763
|
-
assert isinstance(triggers, repositories.Triggers)
|
|
764
|
-
return triggers
|
|
1
|
+
import subprocess
|
|
2
|
+
import inquirer
|
|
3
|
+
import logging
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import jwt
|
|
8
|
+
|
|
9
|
+
from .. import exceptions, entities, repositories, utilities, assets
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(name='dtlpy')
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CommandExecutor:
|
|
15
|
+
|
|
16
|
+
def __init__(self, dl, parser):
|
|
17
|
+
self.dl = dl
|
|
18
|
+
self.parser = parser
|
|
19
|
+
self.utils = Utils(dl)
|
|
20
|
+
|
|
21
|
+
def run(self, args):
|
|
22
|
+
########################
|
|
23
|
+
# Run Command if Exist #
|
|
24
|
+
########################
|
|
25
|
+
if args.operation is None:
|
|
26
|
+
logger.info('See "dlp --help" for options')
|
|
27
|
+
return
|
|
28
|
+
|
|
29
|
+
operation = args.operation.lower().replace('-', '_')
|
|
30
|
+
if hasattr(self, operation):
|
|
31
|
+
getattr(self, operation)(args)
|
|
32
|
+
###############
|
|
33
|
+
# Catch typos #
|
|
34
|
+
###############
|
|
35
|
+
elif args.operation in ["project", 'dataset', 'item', 'service', 'package', 'video', 'deploy', 'generate']:
|
|
36
|
+
self.typos(args=args)
|
|
37
|
+
#######################
|
|
38
|
+
# Catch other options #
|
|
39
|
+
#######################
|
|
40
|
+
elif args.operation:
|
|
41
|
+
print('dlp: "%s" is not an dlp command' % args.operation)
|
|
42
|
+
print('See "dlp --help" for options')
|
|
43
|
+
else:
|
|
44
|
+
print('See "dlp --help" for options')
|
|
45
|
+
|
|
46
|
+
def help(self, args):
|
|
47
|
+
self.parser.print_help()
|
|
48
|
+
|
|
49
|
+
def logout(self, args):
|
|
50
|
+
self.dl.logout()
|
|
51
|
+
logger.info('logout successful')
|
|
52
|
+
|
|
53
|
+
# noinspection PyUnusedLocal
|
|
54
|
+
def login(self, args):
|
|
55
|
+
self.dl.login()
|
|
56
|
+
self.dl.info(with_token=False)
|
|
57
|
+
|
|
58
|
+
def login_token(self, args):
|
|
59
|
+
self.dl.login_token(args.token)
|
|
60
|
+
self.dl.info(with_token=False)
|
|
61
|
+
|
|
62
|
+
def login_api_key(self, args):
|
|
63
|
+
self.dl.login_api_key(api_key=args.api_key)
|
|
64
|
+
|
|
65
|
+
def login_secret(self, args):
|
|
66
|
+
self.login_m2m(args=args)
|
|
67
|
+
|
|
68
|
+
def login_m2m(self, args):
|
|
69
|
+
self.dl.login_m2m(email=args.email,
|
|
70
|
+
password=args.password,
|
|
71
|
+
client_id=args.client_id,
|
|
72
|
+
client_secret=args.client_secret)
|
|
73
|
+
self.dl.info(with_token=False)
|
|
74
|
+
|
|
75
|
+
def upgrade(self, args):
|
|
76
|
+
url = 'dtlpy'
|
|
77
|
+
if args.url is None:
|
|
78
|
+
try:
|
|
79
|
+
payload = jwt.decode(self.dl.client_api.token, algorithms=['HS256'],
|
|
80
|
+
verify=False, options={'verify_signature': False})
|
|
81
|
+
if 'admin' in payload['https://dataloop.ai/authorization']['roles']:
|
|
82
|
+
url = "https://storage.googleapis.com/dtlpy/dev/dtlpy-latest-py3-none-any.whl"
|
|
83
|
+
except Exception:
|
|
84
|
+
pass
|
|
85
|
+
else:
|
|
86
|
+
url = args.url
|
|
87
|
+
|
|
88
|
+
logger.info("Update DTLPy from {}".format(url))
|
|
89
|
+
logger.info("Installing using pip...")
|
|
90
|
+
cmd = "pip install {} --upgrade".format(url)
|
|
91
|
+
subprocess.Popen(cmd, shell=True)
|
|
92
|
+
sys.exit(0)
|
|
93
|
+
|
|
94
|
+
# noinspection PyUnusedLocal
|
|
95
|
+
def init(self, args):
|
|
96
|
+
self.dl.init()
|
|
97
|
+
|
|
98
|
+
# noinspection PyUnusedLocal
|
|
99
|
+
def checkout_state(self, args):
|
|
100
|
+
state = self.dl.checkout_state()
|
|
101
|
+
logger.info('Checked-out:')
|
|
102
|
+
for key, val in state.items():
|
|
103
|
+
try:
|
|
104
|
+
msg = '{entity} name: {name}\t{entity} id: {id}'.format(entity=key, name=val['name'], id=val['id'])
|
|
105
|
+
except KeyError:
|
|
106
|
+
msg = '{entity} Not Found'.format(entity=key)
|
|
107
|
+
logger.info(msg)
|
|
108
|
+
|
|
109
|
+
# noinspection PyUnusedLocal
|
|
110
|
+
def version(self, args):
|
|
111
|
+
logger.info("Dataloop SDK Version: {}".format(self.dl.__version__))
|
|
112
|
+
|
|
113
|
+
def development(self, args):
|
|
114
|
+
if args.development == "local":
|
|
115
|
+
# start the local development
|
|
116
|
+
if args.local == 'start':
|
|
117
|
+
development = self.utils.dl.client_api.state_io.get('development')
|
|
118
|
+
# create default values
|
|
119
|
+
if development is None:
|
|
120
|
+
development = dict()
|
|
121
|
+
if 'port' not in development:
|
|
122
|
+
development['port'] = 5802
|
|
123
|
+
if 'docker_image' not in development:
|
|
124
|
+
development['docker_image'] = 'dataloopai/dtlpy-agent:1.57.3.gpu.cuda11.5.py3.8.opencv'
|
|
125
|
+
|
|
126
|
+
# get values from input
|
|
127
|
+
if args.docker_image is not None:
|
|
128
|
+
development['docker_image'] = args.docker_image
|
|
129
|
+
if args.port is not None:
|
|
130
|
+
development['port'] = int(args.port)
|
|
131
|
+
|
|
132
|
+
# set values to local state
|
|
133
|
+
self.utils.dl.client_api.state_io.put('development', development)
|
|
134
|
+
utilities.local_development.start_session()
|
|
135
|
+
elif args.local == 'pause':
|
|
136
|
+
utilities.local_development.pause_session()
|
|
137
|
+
elif args.local == 'stop':
|
|
138
|
+
utilities.local_development.stop_session()
|
|
139
|
+
else:
|
|
140
|
+
print('Must select one of "start", "pause", "stop". Type "dlp development start --help" for options')
|
|
141
|
+
elif args.development == "remote":
|
|
142
|
+
logger.warning('FUTURE! This is not supported yet..')
|
|
143
|
+
else:
|
|
144
|
+
print('Type "dlp development --help" for options')
|
|
145
|
+
|
|
146
|
+
# noinspection PyUnusedLocal
|
|
147
|
+
|
|
148
|
+
def api(self, args):
|
|
149
|
+
if args.api == "info":
|
|
150
|
+
information = self.dl.info()
|
|
151
|
+
logger.info('-- Dataloop info --')
|
|
152
|
+
_ = [logger.info('{}: {}'.format(key, val)) for key, val in information.items()]
|
|
153
|
+
|
|
154
|
+
if args.api == "setenv":
|
|
155
|
+
self.dl.setenv(args.env)
|
|
156
|
+
logger.info("Platform environment: {}".format(self.dl.environment()))
|
|
157
|
+
|
|
158
|
+
def projects(self, args):
|
|
159
|
+
if args.projects == "ls":
|
|
160
|
+
self.dl.projects.list().print()
|
|
161
|
+
elif args.projects == "web":
|
|
162
|
+
if args.project_name is None:
|
|
163
|
+
args.project_name = self.dl.projects.get().name
|
|
164
|
+
self.dl.projects.open_in_web(project_name=args.project_name)
|
|
165
|
+
elif args.projects == "create":
|
|
166
|
+
project = self.dl.projects.create(args.project_name)
|
|
167
|
+
project.print()
|
|
168
|
+
project.checkout()
|
|
169
|
+
elif args.projects == "checkout":
|
|
170
|
+
self.dl.projects.checkout(project_name=args.project_name)
|
|
171
|
+
else:
|
|
172
|
+
print('Type "dlp projects --help" for options')
|
|
173
|
+
|
|
174
|
+
def datasets(self, args):
|
|
175
|
+
if args.datasets == "ls":
|
|
176
|
+
self.utils.get_datasets_repo(args=args).list().print()
|
|
177
|
+
|
|
178
|
+
elif args.datasets == "checkout":
|
|
179
|
+
self.dl.datasets.checkout(dataset_name=args.dataset_name)
|
|
180
|
+
|
|
181
|
+
elif args.datasets == "web":
|
|
182
|
+
if args.dataset_name is None:
|
|
183
|
+
args.dataset_name = self.dl.datasets.get().name
|
|
184
|
+
self.utils.get_datasets_repo(args=args).open_in_web(dataset_name=args.dataset_name)
|
|
185
|
+
|
|
186
|
+
elif args.datasets == "create":
|
|
187
|
+
dataset = self.utils.get_datasets_repo(args=args).create(dataset_name=args.dataset_name)
|
|
188
|
+
dataset.print()
|
|
189
|
+
if args.checkout:
|
|
190
|
+
dataset.checkout()
|
|
191
|
+
else:
|
|
192
|
+
print('Type "dlp datasets --help" for options')
|
|
193
|
+
|
|
194
|
+
def items(self, args):
|
|
195
|
+
if self.dl.token_expired():
|
|
196
|
+
logger.error("token expired, please login.")
|
|
197
|
+
return
|
|
198
|
+
|
|
199
|
+
if args.items == "ls":
|
|
200
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
201
|
+
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
202
|
+
if isinstance(args.page, str):
|
|
203
|
+
try:
|
|
204
|
+
args.page = int(args.page)
|
|
205
|
+
except ValueError:
|
|
206
|
+
raise ValueError("Input --page must be integer")
|
|
207
|
+
filters = self.dl.Filters()
|
|
208
|
+
|
|
209
|
+
# add filters
|
|
210
|
+
if args.remote_path is not None:
|
|
211
|
+
if isinstance(args.remote_path, list):
|
|
212
|
+
filters.add(field="filename", values=args.remote_path, operator=entities.FiltersOperations.IN)
|
|
213
|
+
else:
|
|
214
|
+
filters.add(field="filename", values=args.remote_path)
|
|
215
|
+
if args.type is not None:
|
|
216
|
+
if isinstance(args.type, list):
|
|
217
|
+
filters.add(field='metadata.system.mimetype', values=args.type,
|
|
218
|
+
operator=entities.FiltersOperations.IN)
|
|
219
|
+
else:
|
|
220
|
+
filters.add(field="metadata.system.mimetype", values=args.type)
|
|
221
|
+
|
|
222
|
+
pages = dataset.items.list(filters=filters, page_offset=args.page)
|
|
223
|
+
pages.print()
|
|
224
|
+
print("Displaying page %d/%d" % (args.page, pages.total_pages_count - 1))
|
|
225
|
+
|
|
226
|
+
elif args.items == "web":
|
|
227
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
228
|
+
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
229
|
+
dataset.items.open_in_web(filepath=args.remote_path)
|
|
230
|
+
|
|
231
|
+
elif args.items == "upload":
|
|
232
|
+
logger.info("Uploading directory...")
|
|
233
|
+
if isinstance(args.file_types, str):
|
|
234
|
+
args.file_types = [t.strip() for t in args.file_types.split(",")]
|
|
235
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
236
|
+
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
237
|
+
|
|
238
|
+
dataset.items.upload(local_path=args.local_path,
|
|
239
|
+
remote_path=args.remote_path,
|
|
240
|
+
file_types=args.file_types,
|
|
241
|
+
overwrite=args.overwrite,
|
|
242
|
+
local_annotations_path=args.local_annotations_path)
|
|
243
|
+
|
|
244
|
+
elif args.items == "download":
|
|
245
|
+
logger.info("Downloading dataset...")
|
|
246
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
247
|
+
dataset = project.datasets.get(dataset_name=args.dataset_name)
|
|
248
|
+
|
|
249
|
+
annotation_options = None
|
|
250
|
+
if args.annotation_options is not None:
|
|
251
|
+
annotation_options = [t.strip() for t in args.annotation_options.split(",")]
|
|
252
|
+
|
|
253
|
+
annotation_filters = None
|
|
254
|
+
if args.annotation_filter_type is not None or args.annotation_filter_label is not None:
|
|
255
|
+
annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
|
|
256
|
+
if args.annotation_filter_type is not None:
|
|
257
|
+
annotation_filter_type = [t.strip() for t in args.annotation_filter_type.split(",")]
|
|
258
|
+
annotation_filters.add(field='type',
|
|
259
|
+
values=annotation_filter_type,
|
|
260
|
+
operator=entities.FiltersOperations.IN)
|
|
261
|
+
if args.annotation_filter_label is not None:
|
|
262
|
+
annotation_filter_label = [t.strip() for t in args.annotation_filter_label.split(",")]
|
|
263
|
+
annotation_filters.add(field='label',
|
|
264
|
+
values=annotation_filter_label,
|
|
265
|
+
operator=entities.FiltersOperations.IN)
|
|
266
|
+
# create remote path filters
|
|
267
|
+
filters = self.dl.Filters()
|
|
268
|
+
if args.remote_path is not None:
|
|
269
|
+
remote_path = [t.strip() for t in args.remote_path.split(",")]
|
|
270
|
+
if len(remote_path) == 1:
|
|
271
|
+
remote_path = remote_path[0]
|
|
272
|
+
filters.add(field="filename", values=remote_path)
|
|
273
|
+
elif len(remote_path) > 1:
|
|
274
|
+
for item in remote_path:
|
|
275
|
+
if '*' in item:
|
|
276
|
+
filters.add(field="dir", values=item, method='or')
|
|
277
|
+
remote_path.pop(remote_path.index(item))
|
|
278
|
+
filters.add(field="dir", values=remote_path, operator=entities.FiltersOperations.IN, method='or')
|
|
279
|
+
|
|
280
|
+
if not args.without_binaries:
|
|
281
|
+
dataset.items.download(filters=filters,
|
|
282
|
+
local_path=args.local_path,
|
|
283
|
+
annotation_options=annotation_options,
|
|
284
|
+
annotation_filters=annotation_filters,
|
|
285
|
+
overwrite=args.overwrite,
|
|
286
|
+
with_text=args.with_text,
|
|
287
|
+
thickness=int(args.thickness),
|
|
288
|
+
to_items_folder=not args.not_items_folder)
|
|
289
|
+
else:
|
|
290
|
+
dataset.download_annotations(filters=filters,
|
|
291
|
+
local_path=args.local_path,
|
|
292
|
+
annotation_options=annotation_options,
|
|
293
|
+
annotation_filters=annotation_filters,
|
|
294
|
+
overwrite=args.overwrite,
|
|
295
|
+
with_text=args.with_text,
|
|
296
|
+
thickness=int(args.thickness))
|
|
297
|
+
|
|
298
|
+
else:
|
|
299
|
+
print('Type "dlp items --help" for options')
|
|
300
|
+
|
|
301
|
+
def videos(self, args):
|
|
302
|
+
if self.dl.token_expired():
|
|
303
|
+
print("[ERROR] token expired, please login.")
|
|
304
|
+
return
|
|
305
|
+
|
|
306
|
+
if args.videos == "play":
|
|
307
|
+
from dtlpy.utilities.videos.video_player import VideoPlayer
|
|
308
|
+
|
|
309
|
+
VideoPlayer(
|
|
310
|
+
item_filepath=args.item_path,
|
|
311
|
+
dataset_name=args.dataset_name,
|
|
312
|
+
project_name=args.project_name,
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
elif args.videos == "upload":
|
|
316
|
+
if (
|
|
317
|
+
(args.split_chunks is not None)
|
|
318
|
+
or (args.split_seconds is not None)
|
|
319
|
+
or (args.split_times is not None)
|
|
320
|
+
):
|
|
321
|
+
# upload with split
|
|
322
|
+
if isinstance(args.split_chunks, str):
|
|
323
|
+
args.split_chunks = int(args.split_chunks)
|
|
324
|
+
if isinstance(args.split_seconds, str):
|
|
325
|
+
args.split_seconds = int(args.split_seconds)
|
|
326
|
+
if isinstance(args.split_times, str):
|
|
327
|
+
args.split_times = [int(sec) for sec in args.split_times.split(",")]
|
|
328
|
+
self.dl.utilities.videos.Videos.split_and_upload(
|
|
329
|
+
project_name=args.project_name,
|
|
330
|
+
dataset_name=args.dataset_name,
|
|
331
|
+
filepath=args.filename,
|
|
332
|
+
remote_path=args.remote_path,
|
|
333
|
+
split_chunks=args.split_chunks,
|
|
334
|
+
split_seconds=args.split_seconds,
|
|
335
|
+
split_pairs=args.split_times,
|
|
336
|
+
)
|
|
337
|
+
else:
|
|
338
|
+
print('Type "dlp videos --help" for options')
|
|
339
|
+
|
|
340
|
+
def services(self, args):
|
|
341
|
+
if self.dl.token_expired():
|
|
342
|
+
print("[ERROR] token expired, please login.")
|
|
343
|
+
return
|
|
344
|
+
|
|
345
|
+
elif args.services == "delete":
|
|
346
|
+
service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
|
|
347
|
+
service.delete()
|
|
348
|
+
logger.info('Service: "{}" deleted successfully'.format(service.name))
|
|
349
|
+
|
|
350
|
+
elif args.services == "ls":
|
|
351
|
+
self.utils.get_services_repo(args=args).list().print()
|
|
352
|
+
elif args.services == "log":
|
|
353
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
354
|
+
service = project.services.get(service_name=args.service_name)
|
|
355
|
+
|
|
356
|
+
logs = service.log(start=args.start)
|
|
357
|
+
try:
|
|
358
|
+
for log in logs:
|
|
359
|
+
if isinstance(log, list):
|
|
360
|
+
for log_record in log:
|
|
361
|
+
print(log_record)
|
|
362
|
+
else:
|
|
363
|
+
print(log)
|
|
364
|
+
except KeyboardInterrupt:
|
|
365
|
+
pass
|
|
366
|
+
|
|
367
|
+
elif args.services == "execute":
|
|
368
|
+
try:
|
|
369
|
+
if args.service_name:
|
|
370
|
+
service = self.utils.get_services_repo(args=args).get(service_name=args.service_name)
|
|
371
|
+
else:
|
|
372
|
+
service = self.dl.services.get()
|
|
373
|
+
except Exception:
|
|
374
|
+
logger.info('Please checkout a service')
|
|
375
|
+
return
|
|
376
|
+
|
|
377
|
+
if args.annotation_id is not None:
|
|
378
|
+
resource = 'Annotation'
|
|
379
|
+
elif args.item_id is not None:
|
|
380
|
+
resource = 'Item'
|
|
381
|
+
elif args.dataset_id is not None:
|
|
382
|
+
resource = 'Dataset'
|
|
383
|
+
else:
|
|
384
|
+
resource = None
|
|
385
|
+
|
|
386
|
+
try:
|
|
387
|
+
execution_input = json.loads(args.inputs)
|
|
388
|
+
except Exception:
|
|
389
|
+
logger.info('Input should be json serializable')
|
|
390
|
+
return
|
|
391
|
+
if len(execution_input) == 0:
|
|
392
|
+
execution_input = None
|
|
393
|
+
|
|
394
|
+
logger.info('executing')
|
|
395
|
+
service.execute(sync=not args.asynchronous,
|
|
396
|
+
execution_input=execution_input,
|
|
397
|
+
function_name=args.function_name,
|
|
398
|
+
resource=resource,
|
|
399
|
+
item_id=args.item_id,
|
|
400
|
+
dataset_id=args.dataset_id,
|
|
401
|
+
annotation_id=args.annotation_id)
|
|
402
|
+
logger.info("Successfully executed function: {}".format(args.function_name))
|
|
403
|
+
|
|
404
|
+
else:
|
|
405
|
+
logger.info('Type "dlp packages --help" for options')
|
|
406
|
+
|
|
407
|
+
def deploy(self, args):
|
|
408
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
409
|
+
json_filepath = args.json_file
|
|
410
|
+
deployed_services, package = self.dl.packages.deploy_from_file(project=project, json_filepath=json_filepath)
|
|
411
|
+
logger.info("Successfully deployed {} from file: {}\nServices: {}".format(len(deployed_services),
|
|
412
|
+
json_filepath,
|
|
413
|
+
[s.name for s in deployed_services]))
|
|
414
|
+
|
|
415
|
+
def generate(self, args):
|
|
416
|
+
package_type = args.package_type if args.package_type else self.dl.PackageCatalog.DEFAULT_PACKAGE_TYPE
|
|
417
|
+
self.dl.packages.generate(name=args.package_name, src_path=os.getcwd(), package_type=package_type)
|
|
418
|
+
self.utils.dl.client_api.state_io.put('package', {'name': args.package_name})
|
|
419
|
+
logger.info('Successfully generated package files')
|
|
420
|
+
|
|
421
|
+
def triggers(self, args):
|
|
422
|
+
|
|
423
|
+
if args.triggers == "create":
|
|
424
|
+
args.actions = [t.strip() for t in args.actions.split(",")]
|
|
425
|
+
try:
|
|
426
|
+
service = self.utils.get_services_repo(args).get(service_name=args.service_name)
|
|
427
|
+
except exceptions.NotFound:
|
|
428
|
+
logger.critical('Service not found. Please check-out a service or provide valid service name')
|
|
429
|
+
return
|
|
430
|
+
|
|
431
|
+
trigger = service.triggers.create(name=args.name,
|
|
432
|
+
filters=json.loads('{}'.format(args.filters.replace("'", '"'))),
|
|
433
|
+
function_name=args.function_name,
|
|
434
|
+
resource=args.resource,
|
|
435
|
+
actions=args.actions)
|
|
436
|
+
logger.info('Trigger created successfully: {}'.format(trigger.name))
|
|
437
|
+
|
|
438
|
+
elif args.triggers == "delete":
|
|
439
|
+
triggers = self.utils.get_triggers_repo(args=args)
|
|
440
|
+
triggers.get(trigger_name=args.trigger_name).delete()
|
|
441
|
+
logger.info('Trigger deleted successfully: {}'.format(args.trigger_name))
|
|
442
|
+
elif args.triggers == "ls":
|
|
443
|
+
triggers = self.utils.get_triggers_repo(args=args)
|
|
444
|
+
triggers.list().print()
|
|
445
|
+
else:
|
|
446
|
+
logger.info('Type "dlp packages --help" for options')
|
|
447
|
+
|
|
448
|
+
def packages(self, args):
|
|
449
|
+
if self.dl.token_expired():
|
|
450
|
+
logger.error("token expired, please login.")
|
|
451
|
+
return
|
|
452
|
+
|
|
453
|
+
elif args.packages == "delete":
|
|
454
|
+
package = self.utils.get_packages_repo(args=args).get(package_name=args.package_name)
|
|
455
|
+
package.delete()
|
|
456
|
+
logger.info('Successfully deleted package {}'.format(package.name))
|
|
457
|
+
|
|
458
|
+
elif args.packages == "ls":
|
|
459
|
+
self.utils.get_packages_repo(args=args).list().print()
|
|
460
|
+
|
|
461
|
+
elif args.packages == "checkout":
|
|
462
|
+
self.dl.packages.checkout(package_name=args.package_name)
|
|
463
|
+
|
|
464
|
+
elif args.packages == "push":
|
|
465
|
+
packages = self.utils.get_packages_repo(args=args)
|
|
466
|
+
|
|
467
|
+
package = packages.push(src_path=args.src_path,
|
|
468
|
+
package_name=args.package_name,
|
|
469
|
+
checkout=args.checkout)
|
|
470
|
+
|
|
471
|
+
logger.info("Successfully pushed package to platform\n"
|
|
472
|
+
"Package id:{}\nPackage version:{}".format(package.id,
|
|
473
|
+
package.version))
|
|
474
|
+
elif args.packages == "deploy":
|
|
475
|
+
packages = self.utils.get_packages_repo(args=args)
|
|
476
|
+
|
|
477
|
+
package = packages.deploy(package_name=args.package_name,
|
|
478
|
+
checkout=args.checkout,
|
|
479
|
+
module_name=args.module_name)
|
|
480
|
+
|
|
481
|
+
logger.info("Successfully pushed package to platform\n"
|
|
482
|
+
"Package id:{}\nPackage version:{}".format(package.id,
|
|
483
|
+
package.version))
|
|
484
|
+
|
|
485
|
+
elif args.packages == "test":
|
|
486
|
+
go_back = False
|
|
487
|
+
if 'src' in os.listdir(os.getcwd()):
|
|
488
|
+
go_back = True
|
|
489
|
+
os.chdir('./src')
|
|
490
|
+
try:
|
|
491
|
+
self.dl.packages.test_local_package(concurrency=int(args.concurrency),
|
|
492
|
+
function_name=args.function_name)
|
|
493
|
+
except Exception:
|
|
494
|
+
logger.exception('failed during test')
|
|
495
|
+
finally:
|
|
496
|
+
if go_back:
|
|
497
|
+
os.chdir('..')
|
|
498
|
+
|
|
499
|
+
else:
|
|
500
|
+
logger.info('Type "dlp packages --help" for options')
|
|
501
|
+
|
|
502
|
+
def app(self, args):
|
|
503
|
+
if args.app == 'pack':
|
|
504
|
+
path = self.dl.dpks.pack()
|
|
505
|
+
logger.info(f'Packed to {path}')
|
|
506
|
+
elif args.app == 'init':
|
|
507
|
+
app_filename = assets.paths.APP_JSON_FILENAME
|
|
508
|
+
if os.path.isfile(app_filename):
|
|
509
|
+
questions = [
|
|
510
|
+
inquirer.Confirm(name='overwrite',
|
|
511
|
+
message=f"Dataloop app already initialized. Re-initialize?",
|
|
512
|
+
default=False)]
|
|
513
|
+
answers = inquirer.prompt(questions)
|
|
514
|
+
if answers.get('overwrite') is False:
|
|
515
|
+
return
|
|
516
|
+
name = args.name
|
|
517
|
+
description = args.description
|
|
518
|
+
attributes = args.attributes
|
|
519
|
+
icon = args.icon
|
|
520
|
+
scope = args.scope
|
|
521
|
+
as_array = [name, description, attributes, icon, scope]
|
|
522
|
+
if as_array.count(None) == len(as_array): # No one value is initialized
|
|
523
|
+
dir_name = os.path.basename(os.getcwd())
|
|
524
|
+
questions = [
|
|
525
|
+
inquirer.Text(name='name',
|
|
526
|
+
message=f"Enter the name of the app (or press enter for '{dir_name}'):",
|
|
527
|
+
default=dir_name),
|
|
528
|
+
inquirer.Text(name='description',
|
|
529
|
+
message="Enter the description (or enter for empty): "),
|
|
530
|
+
inquirer.Text(name='attributes',
|
|
531
|
+
message="Enter the attributes (an object, or enter for empty): ",
|
|
532
|
+
default=None),
|
|
533
|
+
inquirer.Text(name='icon',
|
|
534
|
+
message="Enter the path to the icon (or enter for empty): "),
|
|
535
|
+
inquirer.Text(name='scope',
|
|
536
|
+
message="Enter the scope (or enter for 'organization'): ",
|
|
537
|
+
default='project'),
|
|
538
|
+
]
|
|
539
|
+
answers = inquirer.prompt(questions)
|
|
540
|
+
name = answers.get('name')
|
|
541
|
+
description = answers.get('description')
|
|
542
|
+
attributes = answers.get('attributes')
|
|
543
|
+
icon = answers.get('icon')
|
|
544
|
+
scope = answers.get('scope')
|
|
545
|
+
|
|
546
|
+
if attributes is not None:
|
|
547
|
+
attributes = json.loads(attributes)
|
|
548
|
+
|
|
549
|
+
self.dl.dpks.init(name=name,
|
|
550
|
+
description=description,
|
|
551
|
+
attributes=attributes,
|
|
552
|
+
icon=icon,
|
|
553
|
+
scope=scope)
|
|
554
|
+
elif args.app == 'add':
|
|
555
|
+
if args.panel is True:
|
|
556
|
+
default_panel_name = "myPanel"
|
|
557
|
+
choices = list(entities.dpk.SlotType)
|
|
558
|
+
|
|
559
|
+
questions = [
|
|
560
|
+
inquirer.Text(name='name',
|
|
561
|
+
message=f"Enter PANEL NAME (or press enter for '{default_panel_name}'): ",
|
|
562
|
+
default='default_panel_name'),
|
|
563
|
+
inquirer.List(name='support_slot_type',
|
|
564
|
+
message="Enter SUPPORTED SLOT TYPE:",
|
|
565
|
+
choices=choices),
|
|
566
|
+
]
|
|
567
|
+
|
|
568
|
+
answers = inquirer.prompt(questions)
|
|
569
|
+
#####
|
|
570
|
+
# create a dir for that panel
|
|
571
|
+
os.makedirs(answers.get('name'), exist_ok=True)
|
|
572
|
+
# dump to dataloop.json
|
|
573
|
+
app_filename = assets.paths.APP_JSON_FILENAME
|
|
574
|
+
if not os.path.isfile(app_filename):
|
|
575
|
+
logger.error(f"Can't find app config file ({app_filename}), please run `dlp app init` first")
|
|
576
|
+
else:
|
|
577
|
+
with open(app_filename, 'r') as f:
|
|
578
|
+
dpk = entities.Dpk.from_json(json.load(f))
|
|
579
|
+
dpk.components.panels.append(entities.Panel(name=answers.get('name'),
|
|
580
|
+
supported_slots=[answers.get('support_slot_type')]))
|
|
581
|
+
with open(app_filename, 'w') as f:
|
|
582
|
+
json.dump(dpk.to_json(), f, indent=4)
|
|
583
|
+
|
|
584
|
+
elif args.app == 'publish':
|
|
585
|
+
dpk = self.utils.get_dpks_repo(args).publish()
|
|
586
|
+
if dpk:
|
|
587
|
+
logger.info(f'Published the application: id={dpk.id}')
|
|
588
|
+
else:
|
|
589
|
+
logger.info("Couldn't publish the application")
|
|
590
|
+
elif args.app == 'update':
|
|
591
|
+
# TODO: I think it changed, not implemented
|
|
592
|
+
logger.info('App updated successfully')
|
|
593
|
+
elif args.app == 'resume':
|
|
594
|
+
succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
|
|
595
|
+
if succeed is True:
|
|
596
|
+
logger.info('Resumed application successfully')
|
|
597
|
+
else:
|
|
598
|
+
logger.info('Application resume failed')
|
|
599
|
+
elif args.app == 'pause':
|
|
600
|
+
succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
|
|
601
|
+
if succeed is True:
|
|
602
|
+
logger.info('Paused application successfully')
|
|
603
|
+
else:
|
|
604
|
+
logger.info('Application pause failed')
|
|
605
|
+
elif args.app == 'install':
|
|
606
|
+
app = self.utils.get_apps_repo(args).install(
|
|
607
|
+
dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
|
|
608
|
+
organization_id=args.org_id
|
|
609
|
+
)
|
|
610
|
+
if app is not None:
|
|
611
|
+
logger.info('App installed successfully')
|
|
612
|
+
else:
|
|
613
|
+
logger.info("Couldn't install the app")
|
|
614
|
+
elif args.app == 'pull':
|
|
615
|
+
succeed = self.dl.dpks.pull(dpk_name=args.dpk_name)
|
|
616
|
+
if succeed is True:
|
|
617
|
+
logger.info("Pulled successfully")
|
|
618
|
+
else:
|
|
619
|
+
logger.info("Couldn't pull")
|
|
620
|
+
elif args.app == 'list':
|
|
621
|
+
self.dl.apps.list().print()
|
|
622
|
+
|
|
623
|
+
# noinspection PyUnusedLocal
|
|
624
|
+
@staticmethod
|
|
625
|
+
def pwd(args):
|
|
626
|
+
print(os.getcwd())
|
|
627
|
+
|
|
628
|
+
@staticmethod
|
|
629
|
+
def cd(args):
|
|
630
|
+
directory = args.dir
|
|
631
|
+
if directory == '..':
|
|
632
|
+
directory = os.path.split(os.getcwd())[0]
|
|
633
|
+
os.chdir(directory)
|
|
634
|
+
print(os.getcwd())
|
|
635
|
+
|
|
636
|
+
@staticmethod
|
|
637
|
+
def mkdir(args):
|
|
638
|
+
os.mkdir(args.name)
|
|
639
|
+
|
|
640
|
+
# noinspection PyUnusedLocal
|
|
641
|
+
@staticmethod
|
|
642
|
+
def ls(args):
|
|
643
|
+
print(os.getcwd())
|
|
644
|
+
dirs = os.listdir(os.getcwd())
|
|
645
|
+
import pprint
|
|
646
|
+
|
|
647
|
+
pp = pprint.PrettyPrinter(indent=3)
|
|
648
|
+
pp.pprint(dirs)
|
|
649
|
+
|
|
650
|
+
# noinspection PyUnusedLocal
|
|
651
|
+
@staticmethod
|
|
652
|
+
def clear(args):
|
|
653
|
+
if os.name == "nt":
|
|
654
|
+
os.system('cls')
|
|
655
|
+
else:
|
|
656
|
+
os.system('clear')
|
|
657
|
+
|
|
658
|
+
@staticmethod
|
|
659
|
+
def typos(args):
|
|
660
|
+
print('dlp: "{op}" is not an dlp command. Did you mean "{op}s"?'.format(op=args.operation))
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
class Utils:
|
|
664
|
+
def __init__(self, dl):
|
|
665
|
+
self.dl = dl
|
|
666
|
+
|
|
667
|
+
def get_packages_repo(self, args):
|
|
668
|
+
if args.project_name is not None:
|
|
669
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
670
|
+
packages = project.packages
|
|
671
|
+
else:
|
|
672
|
+
try:
|
|
673
|
+
packages = self.dl.projects.get().packages
|
|
674
|
+
except Exception:
|
|
675
|
+
packages = self.dl.packages
|
|
676
|
+
|
|
677
|
+
assert isinstance(packages, repositories.Packages)
|
|
678
|
+
return packages
|
|
679
|
+
|
|
680
|
+
def get_datasets_repo(self, args):
|
|
681
|
+
if args.project_name is not None:
|
|
682
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
683
|
+
datasets = project.datasets
|
|
684
|
+
else:
|
|
685
|
+
try:
|
|
686
|
+
datasets = self.dl.projects.get().datasets
|
|
687
|
+
except Exception:
|
|
688
|
+
datasets = self.dl.datasets
|
|
689
|
+
|
|
690
|
+
assert isinstance(datasets, repositories.Datasets)
|
|
691
|
+
return datasets
|
|
692
|
+
|
|
693
|
+
def get_services_repo(self, args):
|
|
694
|
+
if args.project_name is not None:
|
|
695
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
696
|
+
packages = project.packages
|
|
697
|
+
services = project.services
|
|
698
|
+
else:
|
|
699
|
+
try:
|
|
700
|
+
packages = self.dl.projects.get().packages
|
|
701
|
+
services = self.dl.projects.get().services
|
|
702
|
+
except Exception:
|
|
703
|
+
packages = self.dl.packages
|
|
704
|
+
services = self.dl.services
|
|
705
|
+
|
|
706
|
+
if args.package_name is not None:
|
|
707
|
+
package = packages.get(package_name=args.package_name)
|
|
708
|
+
services = package.services
|
|
709
|
+
|
|
710
|
+
assert isinstance(services, repositories.Services)
|
|
711
|
+
return services
|
|
712
|
+
|
|
713
|
+
def get_apps_repo(self, args) -> repositories.Apps:
|
|
714
|
+
if args.project_name is not None:
|
|
715
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
716
|
+
apps = project.apps
|
|
717
|
+
else:
|
|
718
|
+
try:
|
|
719
|
+
apps = self.dl.projects.get().apps
|
|
720
|
+
except Exception:
|
|
721
|
+
apps = self.dl.apps
|
|
722
|
+
assert isinstance(apps, repositories.Apps)
|
|
723
|
+
return apps
|
|
724
|
+
|
|
725
|
+
def get_dpks_repo(self, args) -> repositories.Dpks:
|
|
726
|
+
if args.project_name is None and args.project_id is None:
|
|
727
|
+
try:
|
|
728
|
+
dpks = self.dl.projects.get().dpks
|
|
729
|
+
except Exception:
|
|
730
|
+
dpks = self.dl.dpks
|
|
731
|
+
else:
|
|
732
|
+
project = self.dl.projects.get(project_name=args.project_name, project_id=args.project_id)
|
|
733
|
+
dpks = project.dpks
|
|
734
|
+
if dpks.project is None:
|
|
735
|
+
raise ValueError("Must input one of `project-name` or `project-id`") from None
|
|
736
|
+
assert isinstance(dpks, repositories.Dpks)
|
|
737
|
+
return dpks
|
|
738
|
+
|
|
739
|
+
def get_triggers_repo(self, args):
|
|
740
|
+
if args.project_name is not None:
|
|
741
|
+
project = self.dl.projects.get(project_name=args.project_name)
|
|
742
|
+
packages = project.packages
|
|
743
|
+
services = project.services
|
|
744
|
+
triggers = project.triggers
|
|
745
|
+
else:
|
|
746
|
+
try:
|
|
747
|
+
packages = self.dl.projects.get().packages
|
|
748
|
+
services = self.dl.projects.get().services
|
|
749
|
+
triggers = self.dl.projects.get().triggers
|
|
750
|
+
except Exception:
|
|
751
|
+
packages = self.dl.packages
|
|
752
|
+
services = self.dl.services
|
|
753
|
+
triggers = self.dl.triggers
|
|
754
|
+
|
|
755
|
+
if args.package_name is not None:
|
|
756
|
+
package = packages.package_name(name=args.package_name)
|
|
757
|
+
services = package.services
|
|
758
|
+
|
|
759
|
+
if args.service_name is not None:
|
|
760
|
+
service = services.get(service_name=args.service_name)
|
|
761
|
+
triggers = service.triggers
|
|
762
|
+
|
|
763
|
+
assert isinstance(triggers, repositories.Triggers)
|
|
764
|
+
return triggers
|