development-framework-cli 1.7.3__py3-none-any.whl → 1.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/METADATA +3 -1
- development_framework_cli-1.8.0.dist-info/RECORD +66 -0
- {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/WHEEL +1 -1
- dm_cli/__init__.py +1 -1
- dm_cli/cli.py +7 -2
- dm_cli/command_group/data_source.py +6 -5
- dm_cli/command_group/entities.py +51 -7
- dm_cli/dmss.py +57 -2
- dm_cli/dmss_api/__init__.py +103 -16
- dm_cli/dmss_api/api/__init__.py +20 -3
- dm_cli/dmss_api/api/access_control_api.py +604 -311
- dm_cli/dmss_api/api/attribute_api.py +306 -147
- dm_cli/dmss_api/api/blob_api.py +616 -305
- dm_cli/dmss_api/api/blueprint_api.py +577 -288
- dm_cli/dmss_api/api/datasource_api.py +843 -400
- dm_cli/dmss_api/api/default_api.py +9622 -4307
- dm_cli/dmss_api/api/document_api.py +2132 -878
- dm_cli/dmss_api/api/entity_api.py +873 -416
- dm_cli/dmss_api/api/export_api.py +559 -275
- dm_cli/dmss_api/api/file_api.py +331 -169
- dm_cli/dmss_api/api/health_check_api.py +271 -132
- dm_cli/dmss_api/api/lookup_table_api.py +854 -282
- dm_cli/dmss_api/api/meta_api.py +303 -157
- dm_cli/dmss_api/api/personal_access_token_api.py +830 -385
- dm_cli/dmss_api/api/search_api.py +336 -155
- dm_cli/dmss_api/api/whoami_api.py +272 -135
- dm_cli/dmss_api/api_client.py +585 -662
- dm_cli/dmss_api/api_response.py +21 -0
- dm_cli/dmss_api/configuration.py +296 -127
- dm_cli/dmss_api/exceptions.py +90 -30
- dm_cli/dmss_api/models/__init__.py +31 -26
- dm_cli/dmss_api/models/access_control_list.py +95 -0
- dm_cli/dmss_api/models/access_level.py +38 -0
- dm_cli/dmss_api/models/data_source_information.py +104 -0
- dm_cli/dmss_api/models/data_source_request.py +103 -0
- dm_cli/dmss_api/models/dependency.py +103 -0
- dm_cli/dmss_api/models/entity.py +109 -0
- dm_cli/dmss_api/models/error_response.py +96 -0
- dm_cli/dmss_api/models/export_meta_response.py +100 -0
- dm_cli/dmss_api/models/get_attribute_response.py +90 -0
- dm_cli/dmss_api/models/get_blueprint_response.py +92 -0
- dm_cli/dmss_api/models/lookup.py +143 -0
- dm_cli/dmss_api/models/pat_data.py +105 -0
- dm_cli/dmss_api/models/recipe.py +126 -0
- dm_cli/dmss_api/models/recipe_attribute.py +136 -0
- dm_cli/dmss_api/models/repository.py +159 -0
- dm_cli/dmss_api/models/storage_attribute.py +99 -0
- dm_cli/dmss_api/models/storage_data_types.py +40 -0
- dm_cli/dmss_api/models/storage_recipe.py +110 -0
- dm_cli/dmss_api/py.typed +0 -0
- dm_cli/dmss_api/rest.py +198 -243
- dm_cli/import_entity.py +72 -7
- dm_cli/import_package.py +103 -24
- dm_cli/utils/utils.py +9 -0
- development_framework_cli-1.7.3.dist-info/RECORD +0 -71
- dm_cli/dmss_api/api/reference_api.py +0 -174
- dm_cli/dmss_api/apis/__init__.py +0 -32
- dm_cli/dmss_api/model/__init__.py +0 -5
- dm_cli/dmss_api/model/access_control_list.py +0 -287
- dm_cli/dmss_api/model/access_level.py +0 -290
- dm_cli/dmss_api/model/acl.py +0 -287
- dm_cli/dmss_api/model/data_source_information.py +0 -282
- dm_cli/dmss_api/model/data_source_request.py +0 -281
- dm_cli/dmss_api/model/dependency.py +0 -294
- dm_cli/dmss_api/model/entity.py +0 -275
- dm_cli/dmss_api/model/error_response.py +0 -278
- dm_cli/dmss_api/model/export_meta_response.py +0 -277
- dm_cli/dmss_api/model/get_blueprint_response.py +0 -280
- dm_cli/dmss_api/model/lookup.py +0 -283
- dm_cli/dmss_api/model/pat_data.py +0 -299
- dm_cli/dmss_api/model/recipe.py +0 -311
- dm_cli/dmss_api/model/recipe_attribute.py +0 -304
- dm_cli/dmss_api/model/reference.py +0 -280
- dm_cli/dmss_api/model/reference_entity.py +0 -288
- dm_cli/dmss_api/model/repository.py +0 -317
- dm_cli/dmss_api/model/repository_type.py +0 -290
- dm_cli/dmss_api/model/storage_attribute.py +0 -295
- dm_cli/dmss_api/model/storage_data_types.py +0 -292
- dm_cli/dmss_api/model/storage_recipe.py +0 -293
- dm_cli/dmss_api/model_utils.py +0 -2057
- {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/entry_points.txt +0 -0
- {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/licenses/LICENSE +0 -0
{development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: development-framework-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.0
|
|
4
4
|
Summary: CLI tool for working with Data Modelling Storage Service
|
|
5
5
|
Project-URL: Homepage, https://github.com/equinor/dm-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/equinor/dm-cli
|
|
@@ -13,12 +13,14 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
13
13
|
Requires-Python: >=3.13
|
|
14
14
|
Requires-Dist: emoji>=2.15.0
|
|
15
15
|
Requires-Dist: frozendict>=2.4.7
|
|
16
|
+
Requires-Dist: pydantic>=2.11
|
|
16
17
|
Requires-Dist: python-dateutil>=2.9.0.post0
|
|
17
18
|
Requires-Dist: requests>=2.32.5
|
|
18
19
|
Requires-Dist: tenacity>=9.1.4
|
|
19
20
|
Requires-Dist: tqdm>=4.67.3
|
|
20
21
|
Requires-Dist: typer[all]>=0.24.1
|
|
21
22
|
Requires-Dist: typing-extensions>=4.15.0
|
|
23
|
+
Requires-Dist: urllib3>=2.6.3
|
|
22
24
|
Provides-Extra: dev
|
|
23
25
|
Requires-Dist: pre-commit>=4.5.1; extra == 'dev'
|
|
24
26
|
Requires-Dist: pytest>=9.0.2; extra == 'dev'
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
dm_cli/__init__.py,sha256=9YY96-1GuXPdzOeSEWKmn9tAZ72Fc9sM8JlJufdz18w,46
|
|
2
|
+
dm_cli/cli.py,sha256=giAVltKTVUB3MuG8KQcoDTJYTJYcnfICJtAsRVcKOhg,7121
|
|
3
|
+
dm_cli/dmss.py,sha256=bA_jU9lIdY7HizK1DETZViufiWlcmYyFhBXLvFadyeE,5981
|
|
4
|
+
dm_cli/domain.py,sha256=BMBK-WOQwBvDBEr_AoywdXAUPXMHIKIZLnMHpkuzqPs,5650
|
|
5
|
+
dm_cli/enums.py,sha256=7PFPV1u5mexc-p6s8gG-wb7B5SmAORh5esmPQ-zxUtg,1236
|
|
6
|
+
dm_cli/import_entity.py,sha256=mAqVeZlD-uojG4EySPRkNr5biJhFmMe-ziasUiwujqg,6998
|
|
7
|
+
dm_cli/import_package.py,sha256=3dv0GNjRSI1MSFGeQW4_i1G1VaIvsrvfwvY4wKTFeRY,11559
|
|
8
|
+
dm_cli/package_tree_from_zip.py,sha256=bpr9BoRaYKaowaP-zDif6km7mYJLbbby0f0kZHH6DTc,4624
|
|
9
|
+
dm_cli/state.py,sha256=F2L_0bj0P840WDvCmqR0TCsko0AsVyjiOP8SVKocG7s,198
|
|
10
|
+
dm_cli/command_group/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
dm_cli/command_group/data_source.py,sha256=XnWMiJ-ScGfauFEeQ8IoftnasUg5lXYr-6D7jvdE6II,7671
|
|
12
|
+
dm_cli/command_group/entities.py,sha256=5HVX-8_Hy4wbeUCBult4LMyyAXhg8IG4hTu08gXOsuE,6240
|
|
13
|
+
dm_cli/dmss_api/__init__.py,sha256=YPL3TK0SH515dzVPW7zobuzH-cTo1IiMTMRuf6N5Pig,4607
|
|
14
|
+
dm_cli/dmss_api/api_client.py,sha256=cm6nf1CFBbNxsYypjaEvN3SdPD5mlt6z4IQyUvnVAzY,28750
|
|
15
|
+
dm_cli/dmss_api/api_response.py,sha256=eMxw1mpmJcoGZ3gs9z6jM4oYoZ10Gjk333s9sKxGv7s,652
|
|
16
|
+
dm_cli/dmss_api/configuration.py,sha256=19jfVTBvHAfOLvtJfod2ApojtpcruYoUYN7EQpKEzCw,21504
|
|
17
|
+
dm_cli/dmss_api/exceptions.py,sha256=vuHbzbssMJHdaa7UpJiSSeSn58dgj4ZHhlyaH1ixmwo,6480
|
|
18
|
+
dm_cli/dmss_api/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
dm_cli/dmss_api/rest.py,sha256=9Dm343pteceqPyLtGbevqQPcqzjgOpaLs2EI8gsfoJ4,11217
|
|
20
|
+
dm_cli/dmss_api/api/__init__.py,sha256=Ww_2WJ6AlWtxFMV8wzvoN6jMbGFla5J3ozWxzYlVZA0,975
|
|
21
|
+
dm_cli/dmss_api/api/access_control_api.py,sha256=QE6AUbEkhvK81X2EUhBG8XLXGt-A4KdgyBPZORgKO-c,25983
|
|
22
|
+
dm_cli/dmss_api/api/attribute_api.py,sha256=BWLgXQgkqZqC_Rw0f9J50iYFye1W5J3b6cmvvY95Cf4,13183
|
|
23
|
+
dm_cli/dmss_api/api/blob_api.py,sha256=nh5dUZeav-JtK0FfhOlr2r7Q6_vlgzg6E5jtq6X9XwU,26481
|
|
24
|
+
dm_cli/dmss_api/api/blueprint_api.py,sha256=l_De6Hdl_mBwy0Lu7dse9YJQ0wDMe1RnFaszqJ5tc_o,25308
|
|
25
|
+
dm_cli/dmss_api/api/datasource_api.py,sha256=FXVZ1TwIjJ00KvQsnoxi1SOaBj33psl2DL6-l5HBC64,36861
|
|
26
|
+
dm_cli/dmss_api/api/default_api.py,sha256=_xh6oIfR4i9TG7080dl3OrD0Wb4OfEga1AHsEO89Wy8,412495
|
|
27
|
+
dm_cli/dmss_api/api/document_api.py,sha256=ut2qdFhPeGbM-3_ykNcIbAPaCzLYr_joc8nx27h3pDI,93407
|
|
28
|
+
dm_cli/dmss_api/api/entity_api.py,sha256=2OWWGD7eXFM0DQf2D6N1WVuCcztPHchUPiwqZFcRsmg,39700
|
|
29
|
+
dm_cli/dmss_api/api/export_api.py,sha256=hY4kjOHRIOn0Ok6_ULtyU0l4539Ez9Ek4kUuQPyyYqA,25310
|
|
30
|
+
dm_cli/dmss_api/api/file_api.py,sha256=yvn3Gu2vrdD9adMfOHU7k4qjpnbZHGLdb6QlVR4IvlI,14612
|
|
31
|
+
dm_cli/dmss_api/api/health_check_api.py,sha256=-rTaXnjIRkW1Si_OTa_l_bXbpe4xjWnxBS755GOwr_4,11370
|
|
32
|
+
dm_cli/dmss_api/api/lookup_table_api.py,sha256=-nH7WQVMNYu2Gyt5zt7rMIzg_xEythCASGGKbHOovpk,37258
|
|
33
|
+
dm_cli/dmss_api/api/meta_api.py,sha256=eAC1r2Y6xBrsRrtml3ilWQpQ4nWYjnyU2UEyIs8wus4,13538
|
|
34
|
+
dm_cli/dmss_api/api/personal_access_token_api.py,sha256=iS6CKNRHS-747hK8989o55zU5-_eDWwpUGV4Zi1aALs,34959
|
|
35
|
+
dm_cli/dmss_api/api/search_api.py,sha256=xYWw-BHl_w-V5mudFVAHu9WdmAFIoj41hMRoLreezkc,16895
|
|
36
|
+
dm_cli/dmss_api/api/whoami_api.py,sha256=KQ0zuMVLerwDgGYJN5VXponhkuDIePr7TSaNDV7CJwo,12039
|
|
37
|
+
dm_cli/dmss_api/models/__init__.py,sha256=AKLq-oLWP2-NwwUjEgSQIvawiIg_AJcPBQvyY4wNgAw,1496
|
|
38
|
+
dm_cli/dmss_api/models/access_control_list.py,sha256=6SKARxbN_D0Dei6l5zE1inrSvlJFozkhmgYLihqAa2Q,3046
|
|
39
|
+
dm_cli/dmss_api/models/access_level.py,sha256=ZaH6igJJqkZsPL3e_3anLY7a_5xnEeV2Q1_oa8xeykU,716
|
|
40
|
+
dm_cli/dmss_api/models/data_source_information.py,sha256=fzOEWuEXph7ECL6CeD5PQfK4NHcLRe1rMPlmQ3Qon8c,3091
|
|
41
|
+
dm_cli/dmss_api/models/data_source_request.py,sha256=DI82fgSAMwbmAIzJHJ1C6Coy4RUrmq6MRkeo8TH85EI,3214
|
|
42
|
+
dm_cli/dmss_api/models/dependency.py,sha256=s7Ns5aM1CaI4w3w-ggOcQJQuiTZjeEY3R06gTRKjq2I,3114
|
|
43
|
+
dm_cli/dmss_api/models/entity.py,sha256=akPTBM1Fr7Ny0oGfHO1tL06msxs3fZ8ejbqXx3YnldY,3447
|
|
44
|
+
dm_cli/dmss_api/models/error_response.py,sha256=4szu-dkTXtxSYPaCmkJYeXU6wjOteF3Q0z4NLFCDdmo,3189
|
|
45
|
+
dm_cli/dmss_api/models/export_meta_response.py,sha256=S-66jV8qOW-ZCBhlY2jD4muzxNvMEN1cqJZcA_5oJNE,3306
|
|
46
|
+
dm_cli/dmss_api/models/get_attribute_response.py,sha256=TAieFVQrfuNxR5ilIItRD59GfpM7pG3Qo4JgoUcWu98,2560
|
|
47
|
+
dm_cli/dmss_api/models/get_blueprint_response.py,sha256=66zeRqsnmNeH9e_Xa5FKsff19TGeGXKrxJeXao5S5yw,2772
|
|
48
|
+
dm_cli/dmss_api/models/lookup.py,sha256=Rjo8K_i11YEyL6do0K4HZ3uYFeImggIY8Fr86pjDZsg,5584
|
|
49
|
+
dm_cli/dmss_api/models/pat_data.py,sha256=EC0H-JN1pxSSAGlvnvhaLy5_DonErI1aQyrgW0kmHxo,3257
|
|
50
|
+
dm_cli/dmss_api/models/recipe.py,sha256=pxbRPcv3gi8Ye7DdaNZZhiR2dXGsNxOttffwL7VjrBk,4774
|
|
51
|
+
dm_cli/dmss_api/models/recipe_attribute.py,sha256=UxMB8jHt6Xls5qW73THmFAJF-ntDYEGlCDsAjq21nms,4879
|
|
52
|
+
dm_cli/dmss_api/models/repository.py,sha256=RvKqpdIbb6C6WxYr7RpgtVQRH0Z2OBodKacK_rp6ykc,5626
|
|
53
|
+
dm_cli/dmss_api/models/storage_attribute.py,sha256=_l2_dPWOt8Ewm13kMuybrFneTH-JjXvulpnX9bMzCvI,3334
|
|
54
|
+
dm_cli/dmss_api/models/storage_data_types.py,sha256=CLRbZfQ1v-b5A7XCOp5XBUNDDB2x5tCctlZzeQPnw6U,785
|
|
55
|
+
dm_cli/dmss_api/models/storage_recipe.py,sha256=qlsiNuwM0VDt3yuImGNVZQ9cLzPZ3f6k-q18FFArZKo,3811
|
|
56
|
+
dm_cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
57
|
+
dm_cli/utils/file_structure.py,sha256=efIcl7kr7LGm9-2AKt6M6uXe0L8NHNBg0Icy1xO8rSM,1020
|
|
58
|
+
dm_cli/utils/reference.py,sha256=V11edpj8mJSCbXlWlv8HW2zO4AwHRqDnmwdxKv5_qpA,5593
|
|
59
|
+
dm_cli/utils/resolve_local_ids.py,sha256=I9kXVZTQgqr07eu4CghGQVU99MAEXCdn6hacVX0AYOY,5726
|
|
60
|
+
dm_cli/utils/utils.py,sha256=UEuW3ZfQ4cqNLXqyP-fFRiK1bgmaPdIzvl8Z566cx-s,7345
|
|
61
|
+
dm_cli/utils/zip.py,sha256=Zdq6Z978i_1iDAvbrqZwJNsw4QersVtK0HeqFiRagdA,2781
|
|
62
|
+
development_framework_cli-1.8.0.dist-info/METADATA,sha256=iEUXZdnBCU0KSBGzMay10r2OP9kMcdiMl0yVznl8Wvs,8234
|
|
63
|
+
development_framework_cli-1.8.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
64
|
+
development_framework_cli-1.8.0.dist-info/entry_points.txt,sha256=PAOXUTc_jhfY9Hfs7BNFzh4TlbrOGYU7HgTM-P7h074,38
|
|
65
|
+
development_framework_cli-1.8.0.dist-info/licenses/LICENSE,sha256=zefthAHZDOYIcHUXefkN_Fvuy4s-nYfoLX_dsycVDxc,1064
|
|
66
|
+
development_framework_cli-1.8.0.dist-info/RECORD,,
|
dm_cli/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
VERSION = "1.
|
|
1
|
+
VERSION = "1.8.0" # x-release-please-version
|
dm_cli/cli.py
CHANGED
|
@@ -11,8 +11,9 @@ from typing_extensions import Annotated
|
|
|
11
11
|
|
|
12
12
|
from dm_cli import VERSION
|
|
13
13
|
from dm_cli.command_group.data_source import data_source_app, reset_data_source
|
|
14
|
-
from dm_cli.command_group.entities import entities_app
|
|
14
|
+
from dm_cli.command_group.entities import entities_app
|
|
15
15
|
from dm_cli.dmss import dmss_api, dmss_exception_wrapper, export
|
|
16
|
+
from dm_cli.import_entity import import_folders_as_one_package
|
|
16
17
|
from dm_cli.state import state
|
|
17
18
|
from dm_cli.utils.file_structure import get_app_dir_structure, get_json_files_in_dir
|
|
18
19
|
from dm_cli.utils.utils import (
|
|
@@ -74,7 +75,11 @@ def import_plugin_blueprints(
|
|
|
74
75
|
"""
|
|
75
76
|
state.force = True
|
|
76
77
|
dmss_exception_wrapper(
|
|
77
|
-
|
|
78
|
+
import_folders_as_one_package,
|
|
79
|
+
source_path=Path(path) / "blueprints",
|
|
80
|
+
destination="system/Plugins",
|
|
81
|
+
package_name=Path(path).name,
|
|
82
|
+
validate=validate,
|
|
78
83
|
)
|
|
79
84
|
|
|
80
85
|
|
|
@@ -3,6 +3,7 @@ from pathlib import Path
|
|
|
3
3
|
|
|
4
4
|
import emoji
|
|
5
5
|
import typer
|
|
6
|
+
import urllib3
|
|
6
7
|
from rich import print
|
|
7
8
|
from tenacity import (
|
|
8
9
|
retry,
|
|
@@ -36,7 +37,7 @@ def import_data_source(
|
|
|
36
37
|
wait=wait_random_exponential(multiplier=1, max=60),
|
|
37
38
|
stop=stop_after_attempt(5),
|
|
38
39
|
reraise=True,
|
|
39
|
-
retry=retry_if_exception_type(ServiceException),
|
|
40
|
+
retry=retry_if_exception_type((ServiceException, urllib3.exceptions.HTTPError)),
|
|
40
41
|
)
|
|
41
42
|
def retry_wrapper():
|
|
42
43
|
data_source_path = Path(path)
|
|
@@ -48,14 +49,14 @@ def import_data_source(
|
|
|
48
49
|
# Read the data source definition
|
|
49
50
|
with open(data_source_path) as file:
|
|
50
51
|
document = json.load(file)
|
|
51
|
-
existing_data_sources =
|
|
52
|
-
if any(existing_document
|
|
52
|
+
existing_data_sources = dmss_api.data_source_get_all()
|
|
53
|
+
if any(existing_document.name == document["name"] for existing_document in existing_data_sources):
|
|
53
54
|
print(f"WARNING: data source {document['name']} already exists. Updating existing data source.")
|
|
54
55
|
|
|
55
|
-
|
|
56
|
+
dmss_api.data_source_save(document["name"], document)
|
|
56
57
|
print(f"\tImported data source '{document['name']}' ✓")
|
|
57
58
|
|
|
58
|
-
retry_wrapper
|
|
59
|
+
dmss_exception_wrapper(retry_wrapper)
|
|
59
60
|
|
|
60
61
|
|
|
61
62
|
@data_source_app.command("import-all", help="Import all datasources found in the directory given by 'path'")
|
dm_cli/command_group/entities.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import json
|
|
2
2
|
from pathlib import Path
|
|
3
|
+
from typing import List, Optional
|
|
3
4
|
|
|
4
5
|
import typer
|
|
5
6
|
from rich import print
|
|
@@ -70,24 +71,63 @@ def import_entity(
|
|
|
70
71
|
return dmss_exception_wrapper(inner_import)
|
|
71
72
|
|
|
72
73
|
|
|
74
|
+
@entities_app.command("import-batch")
|
|
75
|
+
def import_batch(
|
|
76
|
+
data_directory: Annotated[
|
|
77
|
+
Path,
|
|
78
|
+
typer.Argument(
|
|
79
|
+
help="Path to a directory laid out as <dataSource>/<rootPackage>. Every root package found is imported into the data source it is located under."
|
|
80
|
+
),
|
|
81
|
+
],
|
|
82
|
+
exclude: Annotated[
|
|
83
|
+
Optional[List[str]],
|
|
84
|
+
typer.Option(help="Name of a data source to skip. Can be repeated."),
|
|
85
|
+
] = None,
|
|
86
|
+
validate: Annotated[bool, typer.Option(help="if True, all entities uploaded will be validated.")] = True,
|
|
87
|
+
) -> bool:
|
|
88
|
+
"""
|
|
89
|
+
Import every root package found under <data_directory> into its data source.
|
|
90
|
+
|
|
91
|
+
This is equivalent to running 'entities import <data_directory>/<dataSource>/<rootPackage> <dataSource>'
|
|
92
|
+
once per root package, but does so in a single process and over a single reused HTTP connection.
|
|
93
|
+
"""
|
|
94
|
+
if not data_directory.is_dir():
|
|
95
|
+
raise FileNotFoundError(f"The path '{data_directory}' is not a directory.")
|
|
96
|
+
|
|
97
|
+
def visible_entries(directory: Path) -> list[Path]:
|
|
98
|
+
# Hidden entries are skipped so that incidental files such as '.DS_Store'
|
|
99
|
+
# are not mistaken for packages.
|
|
100
|
+
return sorted(entry for entry in directory.iterdir() if not entry.name.startswith("."))
|
|
101
|
+
|
|
102
|
+
excluded = set(exclude or [])
|
|
103
|
+
for data_source_dir in visible_entries(data_directory):
|
|
104
|
+
if not data_source_dir.is_dir():
|
|
105
|
+
continue
|
|
106
|
+
if data_source_dir.name in excluded:
|
|
107
|
+
print(f"Skipping data source '{data_source_dir.name}'")
|
|
108
|
+
continue
|
|
109
|
+
for root_package in visible_entries(data_source_dir):
|
|
110
|
+
import_entity(str(root_package), data_source_dir.name, validate)
|
|
111
|
+
return True
|
|
112
|
+
|
|
113
|
+
|
|
73
114
|
@entities_app.command("validate")
|
|
74
115
|
def validate_entity(
|
|
75
|
-
|
|
76
|
-
str,
|
|
116
|
+
destinations: Annotated[
|
|
117
|
+
List[str],
|
|
77
118
|
typer.Argument(
|
|
78
|
-
help="
|
|
119
|
+
help="Addresses for the folders or files to validate. Should be on the format <DataSource>/<rootPackage>/<subPackage>/<entity>"
|
|
79
120
|
),
|
|
80
121
|
],
|
|
81
122
|
) -> bool:
|
|
82
|
-
"""Recursively validate
|
|
83
|
-
print(f"Validating entities recursively in: {destination}")
|
|
123
|
+
"""Recursively validate one or more entities at remote targets"""
|
|
84
124
|
|
|
85
125
|
@retry(
|
|
86
126
|
wait=wait_random_exponential(multiplier=1, max=60),
|
|
87
127
|
stop=stop_after_attempt(5),
|
|
88
128
|
retry=retry_if_not_exception_type((ApplicationException, RuntimeError)),
|
|
89
129
|
)
|
|
90
|
-
def validation_error_wrapper():
|
|
130
|
+
def validation_error_wrapper(destination: str):
|
|
91
131
|
try:
|
|
92
132
|
dmss_api.validate_existing_entity(destination)
|
|
93
133
|
except ApiException as e:
|
|
@@ -97,7 +137,11 @@ def validate_entity(
|
|
|
97
137
|
raise typer.Exit(code=1)
|
|
98
138
|
raise e
|
|
99
139
|
|
|
100
|
-
|
|
140
|
+
for destination in destinations:
|
|
141
|
+
print(f"Validating entities recursively in: {destination}")
|
|
142
|
+
# Validation is CPU bound on the DMSS side, so there is nothing to gain from validating targets
|
|
143
|
+
# concurrently; doing so only makes the blueprint caches thrash.
|
|
144
|
+
dmss_exception_wrapper(validation_error_wrapper, destination)
|
|
101
145
|
|
|
102
146
|
|
|
103
147
|
@entities_app.command("delete")
|
dm_cli/dmss.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import re
|
|
2
3
|
from typing import Any, Callable
|
|
3
4
|
|
|
4
|
-
import requests
|
|
5
5
|
import typer
|
|
6
6
|
from rich.console import Console
|
|
7
7
|
from rich.text import Text
|
|
@@ -14,12 +14,64 @@ from tenacity import (
|
|
|
14
14
|
|
|
15
15
|
from dm_cli.dmss_api import ApiException
|
|
16
16
|
from dm_cli.dmss_api.api.default_api import DefaultApi
|
|
17
|
+
from dm_cli.dmss_api.api_client import ApiClient
|
|
17
18
|
from dm_cli.dmss_api.exceptions import NotFoundException, ServiceException
|
|
18
19
|
from dm_cli.state import state
|
|
19
20
|
|
|
20
21
|
console = Console()
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
# The concrete types a request body loaded from JSON is built from.
|
|
24
|
+
_JSON_TYPES = frozenset({str, int, float, bool, type(None)})
|
|
25
|
+
|
|
26
|
+
# Generated response types that are built only from dicts, lists and untyped values, such as
|
|
27
|
+
# "object", "Dict[str, object]" and "Dict[str, Optional[object]]".
|
|
28
|
+
_PLAIN_JSON_RESPONSE_TYPE = re.compile(r"(?:Dict\[str, |List\[|Optional\[)*object\]*")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class JsonApiClient(ApiClient):
|
|
32
|
+
"""An ApiClient that handles plain JSON without the generated type dispatch.
|
|
33
|
+
|
|
34
|
+
Most DMSS endpoints take and return documents: arbitrarily nested JSON that the generated client
|
|
35
|
+
describes as "object". It still walks every node of every document on the way in and on the way
|
|
36
|
+
out, which is both slow and, for responses, broken.
|
|
37
|
+
|
|
38
|
+
This lives here rather than in dm_cli/dmss_api, because that package is generated by
|
|
39
|
+
'generate-dmss-api.sh' and any edit to it is lost on the next regeneration.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def sanitize_for_serialization(self, obj):
|
|
43
|
+
"""Serialize a request body, dispatching on the concrete type first.
|
|
44
|
+
|
|
45
|
+
Request bodies are nested dicts loaded from JSON, and this recurses over every node of every
|
|
46
|
+
document, so it is called millions of times during a package import. The generated
|
|
47
|
+
implementation tests each value against abstract base classes, which is far more expensive
|
|
48
|
+
than an exact type lookup; dispatching on the concrete type first makes it ~3x faster.
|
|
49
|
+
Anything that is not plain JSON still falls through to the generated implementation.
|
|
50
|
+
"""
|
|
51
|
+
obj_type = type(obj)
|
|
52
|
+
if obj_type in _JSON_TYPES:
|
|
53
|
+
return obj
|
|
54
|
+
if obj_type is dict:
|
|
55
|
+
return {key: self.sanitize_for_serialization(val) for key, val in obj.items()}
|
|
56
|
+
if obj_type is list:
|
|
57
|
+
return [self.sanitize_for_serialization(item) for item in obj]
|
|
58
|
+
return super().sanitize_for_serialization(obj)
|
|
59
|
+
|
|
60
|
+
def deserialize(self, response_text, response_type, content_type):
|
|
61
|
+
"""Deserialize a response, returning plain JSON as-is.
|
|
62
|
+
|
|
63
|
+
The generated deserializer recurses through a type string such as
|
|
64
|
+
"Dict[str, Optional[object]]" only to hand every leaf to a function that returns it
|
|
65
|
+
unchanged, so collapsing the whole type to "object" is exact and skips the walk. It also
|
|
66
|
+
works around a generator bug: openapi-generator 7.24 emits "Optional[...]" but its own
|
|
67
|
+
deserializer cannot parse it, and raises AttributeError on every successful document_get.
|
|
68
|
+
"""
|
|
69
|
+
if isinstance(response_type, str) and _PLAIN_JSON_RESPONSE_TYPE.fullmatch(response_type):
|
|
70
|
+
response_type = "object"
|
|
71
|
+
return super().deserialize(response_text, response_type, content_type)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
dmss_api = DefaultApi(api_client=JsonApiClient())
|
|
23
75
|
|
|
24
76
|
|
|
25
77
|
class ApplicationException(Exception):
|
|
@@ -66,6 +118,9 @@ def export(absolute_document_ref: str):
|
|
|
66
118
|
The reason dmss_api cannot be used directly is that there were some issues with interpreting the JSON schema,
|
|
67
119
|
which caused the export function in the generated DMSS api to not work properly.
|
|
68
120
|
"""
|
|
121
|
+
# Imported lazily; 'requests' is only needed by this command and costs ~200ms to import.
|
|
122
|
+
import requests
|
|
123
|
+
|
|
69
124
|
headers = {"Access-Key": state.token}
|
|
70
125
|
|
|
71
126
|
response = requests.get(f"{state.dmss_url}/api/export/{absolute_document_ref}", headers=headers) # nosec
|
dm_cli/dmss_api/__init__.py
CHANGED
|
@@ -1,27 +1,114 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
1
3
|
# flake8: noqa
|
|
2
4
|
|
|
3
5
|
"""
|
|
4
6
|
Data Modelling Storage Service
|
|
5
7
|
|
|
6
|
-
API for basic data modelling interaction
|
|
8
|
+
API for basic data modelling interaction
|
|
7
9
|
|
|
8
|
-
The version of the OpenAPI document: 1.
|
|
9
|
-
Generated by
|
|
10
|
-
|
|
10
|
+
The version of the OpenAPI document: 1.28.4
|
|
11
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
12
|
+
|
|
13
|
+
Do not edit the class manually.
|
|
14
|
+
""" # noqa: E501
|
|
11
15
|
|
|
12
16
|
|
|
13
17
|
__version__ = "1.0.0"
|
|
14
18
|
|
|
19
|
+
# Define package exports
|
|
20
|
+
__all__ = [
|
|
21
|
+
"AccessControlApi",
|
|
22
|
+
"AttributeApi",
|
|
23
|
+
"BlobApi",
|
|
24
|
+
"BlueprintApi",
|
|
25
|
+
"DatasourceApi",
|
|
26
|
+
"DefaultApi",
|
|
27
|
+
"DocumentApi",
|
|
28
|
+
"EntityApi",
|
|
29
|
+
"ExportApi",
|
|
30
|
+
"FileApi",
|
|
31
|
+
"HealthCheckApi",
|
|
32
|
+
"LookupTableApi",
|
|
33
|
+
"MetaApi",
|
|
34
|
+
"PersonalAccessTokenApi",
|
|
35
|
+
"SearchApi",
|
|
36
|
+
"WhoamiApi",
|
|
37
|
+
"ApiResponse",
|
|
38
|
+
"ApiClient",
|
|
39
|
+
"Configuration",
|
|
40
|
+
"OpenApiException",
|
|
41
|
+
"ApiTypeError",
|
|
42
|
+
"ApiValueError",
|
|
43
|
+
"ApiKeyError",
|
|
44
|
+
"ApiAttributeError",
|
|
45
|
+
"ApiException",
|
|
46
|
+
"AccessControlList",
|
|
47
|
+
"AccessLevel",
|
|
48
|
+
"DataSourceInformation",
|
|
49
|
+
"DataSourceRequest",
|
|
50
|
+
"Dependency",
|
|
51
|
+
"Entity",
|
|
52
|
+
"ErrorResponse",
|
|
53
|
+
"ExportMetaResponse",
|
|
54
|
+
"GetAttributeResponse",
|
|
55
|
+
"GetBlueprintResponse",
|
|
56
|
+
"Lookup",
|
|
57
|
+
"PATData",
|
|
58
|
+
"Recipe",
|
|
59
|
+
"RecipeAttribute",
|
|
60
|
+
"Repository",
|
|
61
|
+
"StorageAttribute",
|
|
62
|
+
"StorageDataTypes",
|
|
63
|
+
"StorageRecipe",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
# import apis into sdk package
|
|
67
|
+
from dm_cli.dmss_api.api.access_control_api import AccessControlApi as AccessControlApi
|
|
68
|
+
from dm_cli.dmss_api.api.attribute_api import AttributeApi as AttributeApi
|
|
69
|
+
from dm_cli.dmss_api.api.blob_api import BlobApi as BlobApi
|
|
70
|
+
from dm_cli.dmss_api.api.blueprint_api import BlueprintApi as BlueprintApi
|
|
71
|
+
from dm_cli.dmss_api.api.datasource_api import DatasourceApi as DatasourceApi
|
|
72
|
+
from dm_cli.dmss_api.api.default_api import DefaultApi as DefaultApi
|
|
73
|
+
from dm_cli.dmss_api.api.document_api import DocumentApi as DocumentApi
|
|
74
|
+
from dm_cli.dmss_api.api.entity_api import EntityApi as EntityApi
|
|
75
|
+
from dm_cli.dmss_api.api.export_api import ExportApi as ExportApi
|
|
76
|
+
from dm_cli.dmss_api.api.file_api import FileApi as FileApi
|
|
77
|
+
from dm_cli.dmss_api.api.health_check_api import HealthCheckApi as HealthCheckApi
|
|
78
|
+
from dm_cli.dmss_api.api.lookup_table_api import LookupTableApi as LookupTableApi
|
|
79
|
+
from dm_cli.dmss_api.api.meta_api import MetaApi as MetaApi
|
|
80
|
+
from dm_cli.dmss_api.api.personal_access_token_api import PersonalAccessTokenApi as PersonalAccessTokenApi
|
|
81
|
+
from dm_cli.dmss_api.api.search_api import SearchApi as SearchApi
|
|
82
|
+
from dm_cli.dmss_api.api.whoami_api import WhoamiApi as WhoamiApi
|
|
83
|
+
|
|
15
84
|
# import ApiClient
|
|
16
|
-
from dm_cli.dmss_api.
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
from dm_cli.dmss_api.
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
from dm_cli.dmss_api.exceptions import
|
|
23
|
-
from dm_cli.dmss_api.exceptions import ApiAttributeError
|
|
24
|
-
from dm_cli.dmss_api.exceptions import
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
from dm_cli.dmss_api.
|
|
85
|
+
from dm_cli.dmss_api.api_response import ApiResponse as ApiResponse
|
|
86
|
+
from dm_cli.dmss_api.api_client import ApiClient as ApiClient
|
|
87
|
+
from dm_cli.dmss_api.configuration import Configuration as Configuration
|
|
88
|
+
from dm_cli.dmss_api.exceptions import OpenApiException as OpenApiException
|
|
89
|
+
from dm_cli.dmss_api.exceptions import ApiTypeError as ApiTypeError
|
|
90
|
+
from dm_cli.dmss_api.exceptions import ApiValueError as ApiValueError
|
|
91
|
+
from dm_cli.dmss_api.exceptions import ApiKeyError as ApiKeyError
|
|
92
|
+
from dm_cli.dmss_api.exceptions import ApiAttributeError as ApiAttributeError
|
|
93
|
+
from dm_cli.dmss_api.exceptions import ApiException as ApiException
|
|
94
|
+
|
|
95
|
+
# import models into sdk package
|
|
96
|
+
from dm_cli.dmss_api.models.access_control_list import AccessControlList as AccessControlList
|
|
97
|
+
from dm_cli.dmss_api.models.access_level import AccessLevel as AccessLevel
|
|
98
|
+
from dm_cli.dmss_api.models.data_source_information import DataSourceInformation as DataSourceInformation
|
|
99
|
+
from dm_cli.dmss_api.models.data_source_request import DataSourceRequest as DataSourceRequest
|
|
100
|
+
from dm_cli.dmss_api.models.dependency import Dependency as Dependency
|
|
101
|
+
from dm_cli.dmss_api.models.entity import Entity as Entity
|
|
102
|
+
from dm_cli.dmss_api.models.error_response import ErrorResponse as ErrorResponse
|
|
103
|
+
from dm_cli.dmss_api.models.export_meta_response import ExportMetaResponse as ExportMetaResponse
|
|
104
|
+
from dm_cli.dmss_api.models.get_attribute_response import GetAttributeResponse as GetAttributeResponse
|
|
105
|
+
from dm_cli.dmss_api.models.get_blueprint_response import GetBlueprintResponse as GetBlueprintResponse
|
|
106
|
+
from dm_cli.dmss_api.models.lookup import Lookup as Lookup
|
|
107
|
+
from dm_cli.dmss_api.models.pat_data import PATData as PATData
|
|
108
|
+
from dm_cli.dmss_api.models.recipe import Recipe as Recipe
|
|
109
|
+
from dm_cli.dmss_api.models.recipe_attribute import RecipeAttribute as RecipeAttribute
|
|
110
|
+
from dm_cli.dmss_api.models.repository import Repository as Repository
|
|
111
|
+
from dm_cli.dmss_api.models.storage_attribute import StorageAttribute as StorageAttribute
|
|
112
|
+
from dm_cli.dmss_api.models.storage_data_types import StorageDataTypes as StorageDataTypes
|
|
113
|
+
from dm_cli.dmss_api.models.storage_recipe import StorageRecipe as StorageRecipe
|
|
114
|
+
|
dm_cli/dmss_api/api/__init__.py
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
#
|
|
1
|
+
# flake8: noqa
|
|
2
|
+
|
|
3
|
+
# import apis into api package
|
|
4
|
+
from dm_cli.dmss_api.api.access_control_api import AccessControlApi
|
|
5
|
+
from dm_cli.dmss_api.api.attribute_api import AttributeApi
|
|
6
|
+
from dm_cli.dmss_api.api.blob_api import BlobApi
|
|
7
|
+
from dm_cli.dmss_api.api.blueprint_api import BlueprintApi
|
|
8
|
+
from dm_cli.dmss_api.api.datasource_api import DatasourceApi
|
|
9
|
+
from dm_cli.dmss_api.api.default_api import DefaultApi
|
|
10
|
+
from dm_cli.dmss_api.api.document_api import DocumentApi
|
|
11
|
+
from dm_cli.dmss_api.api.entity_api import EntityApi
|
|
12
|
+
from dm_cli.dmss_api.api.export_api import ExportApi
|
|
13
|
+
from dm_cli.dmss_api.api.file_api import FileApi
|
|
14
|
+
from dm_cli.dmss_api.api.health_check_api import HealthCheckApi
|
|
15
|
+
from dm_cli.dmss_api.api.lookup_table_api import LookupTableApi
|
|
16
|
+
from dm_cli.dmss_api.api.meta_api import MetaApi
|
|
17
|
+
from dm_cli.dmss_api.api.personal_access_token_api import PersonalAccessTokenApi
|
|
18
|
+
from dm_cli.dmss_api.api.search_api import SearchApi
|
|
19
|
+
from dm_cli.dmss_api.api.whoami_api import WhoamiApi
|
|
20
|
+
|