development-framework-cli 1.7.3__py3-none-any.whl → 1.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/METADATA +3 -1
  2. development_framework_cli-1.8.0.dist-info/RECORD +66 -0
  3. {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/WHEEL +1 -1
  4. dm_cli/__init__.py +1 -1
  5. dm_cli/cli.py +7 -2
  6. dm_cli/command_group/data_source.py +6 -5
  7. dm_cli/command_group/entities.py +51 -7
  8. dm_cli/dmss.py +57 -2
  9. dm_cli/dmss_api/__init__.py +103 -16
  10. dm_cli/dmss_api/api/__init__.py +20 -3
  11. dm_cli/dmss_api/api/access_control_api.py +604 -311
  12. dm_cli/dmss_api/api/attribute_api.py +306 -147
  13. dm_cli/dmss_api/api/blob_api.py +616 -305
  14. dm_cli/dmss_api/api/blueprint_api.py +577 -288
  15. dm_cli/dmss_api/api/datasource_api.py +843 -400
  16. dm_cli/dmss_api/api/default_api.py +9622 -4307
  17. dm_cli/dmss_api/api/document_api.py +2132 -878
  18. dm_cli/dmss_api/api/entity_api.py +873 -416
  19. dm_cli/dmss_api/api/export_api.py +559 -275
  20. dm_cli/dmss_api/api/file_api.py +331 -169
  21. dm_cli/dmss_api/api/health_check_api.py +271 -132
  22. dm_cli/dmss_api/api/lookup_table_api.py +854 -282
  23. dm_cli/dmss_api/api/meta_api.py +303 -157
  24. dm_cli/dmss_api/api/personal_access_token_api.py +830 -385
  25. dm_cli/dmss_api/api/search_api.py +336 -155
  26. dm_cli/dmss_api/api/whoami_api.py +272 -135
  27. dm_cli/dmss_api/api_client.py +585 -662
  28. dm_cli/dmss_api/api_response.py +21 -0
  29. dm_cli/dmss_api/configuration.py +296 -127
  30. dm_cli/dmss_api/exceptions.py +90 -30
  31. dm_cli/dmss_api/models/__init__.py +31 -26
  32. dm_cli/dmss_api/models/access_control_list.py +95 -0
  33. dm_cli/dmss_api/models/access_level.py +38 -0
  34. dm_cli/dmss_api/models/data_source_information.py +104 -0
  35. dm_cli/dmss_api/models/data_source_request.py +103 -0
  36. dm_cli/dmss_api/models/dependency.py +103 -0
  37. dm_cli/dmss_api/models/entity.py +109 -0
  38. dm_cli/dmss_api/models/error_response.py +96 -0
  39. dm_cli/dmss_api/models/export_meta_response.py +100 -0
  40. dm_cli/dmss_api/models/get_attribute_response.py +90 -0
  41. dm_cli/dmss_api/models/get_blueprint_response.py +92 -0
  42. dm_cli/dmss_api/models/lookup.py +143 -0
  43. dm_cli/dmss_api/models/pat_data.py +105 -0
  44. dm_cli/dmss_api/models/recipe.py +126 -0
  45. dm_cli/dmss_api/models/recipe_attribute.py +136 -0
  46. dm_cli/dmss_api/models/repository.py +159 -0
  47. dm_cli/dmss_api/models/storage_attribute.py +99 -0
  48. dm_cli/dmss_api/models/storage_data_types.py +40 -0
  49. dm_cli/dmss_api/models/storage_recipe.py +110 -0
  50. dm_cli/dmss_api/py.typed +0 -0
  51. dm_cli/dmss_api/rest.py +198 -243
  52. dm_cli/import_entity.py +72 -7
  53. dm_cli/import_package.py +103 -24
  54. dm_cli/utils/utils.py +9 -0
  55. development_framework_cli-1.7.3.dist-info/RECORD +0 -71
  56. dm_cli/dmss_api/api/reference_api.py +0 -174
  57. dm_cli/dmss_api/apis/__init__.py +0 -32
  58. dm_cli/dmss_api/model/__init__.py +0 -5
  59. dm_cli/dmss_api/model/access_control_list.py +0 -287
  60. dm_cli/dmss_api/model/access_level.py +0 -290
  61. dm_cli/dmss_api/model/acl.py +0 -287
  62. dm_cli/dmss_api/model/data_source_information.py +0 -282
  63. dm_cli/dmss_api/model/data_source_request.py +0 -281
  64. dm_cli/dmss_api/model/dependency.py +0 -294
  65. dm_cli/dmss_api/model/entity.py +0 -275
  66. dm_cli/dmss_api/model/error_response.py +0 -278
  67. dm_cli/dmss_api/model/export_meta_response.py +0 -277
  68. dm_cli/dmss_api/model/get_blueprint_response.py +0 -280
  69. dm_cli/dmss_api/model/lookup.py +0 -283
  70. dm_cli/dmss_api/model/pat_data.py +0 -299
  71. dm_cli/dmss_api/model/recipe.py +0 -311
  72. dm_cli/dmss_api/model/recipe_attribute.py +0 -304
  73. dm_cli/dmss_api/model/reference.py +0 -280
  74. dm_cli/dmss_api/model/reference_entity.py +0 -288
  75. dm_cli/dmss_api/model/repository.py +0 -317
  76. dm_cli/dmss_api/model/repository_type.py +0 -290
  77. dm_cli/dmss_api/model/storage_attribute.py +0 -295
  78. dm_cli/dmss_api/model/storage_data_types.py +0 -292
  79. dm_cli/dmss_api/model/storage_recipe.py +0 -293
  80. dm_cli/dmss_api/model_utils.py +0 -2057
  81. {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/entry_points.txt +0 -0
  82. {development_framework_cli-1.7.3.dist-info → development_framework_cli-1.8.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: development-framework-cli
3
- Version: 1.7.3
3
+ Version: 1.8.0
4
4
  Summary: CLI tool for working with Data Modelling Storage Service
5
5
  Project-URL: Homepage, https://github.com/equinor/dm-cli
6
6
  Project-URL: Repository, https://github.com/equinor/dm-cli
@@ -13,12 +13,14 @@ Classifier: Programming Language :: Python :: 3.13
13
13
  Requires-Python: >=3.13
14
14
  Requires-Dist: emoji>=2.15.0
15
15
  Requires-Dist: frozendict>=2.4.7
16
+ Requires-Dist: pydantic>=2.11
16
17
  Requires-Dist: python-dateutil>=2.9.0.post0
17
18
  Requires-Dist: requests>=2.32.5
18
19
  Requires-Dist: tenacity>=9.1.4
19
20
  Requires-Dist: tqdm>=4.67.3
20
21
  Requires-Dist: typer[all]>=0.24.1
21
22
  Requires-Dist: typing-extensions>=4.15.0
23
+ Requires-Dist: urllib3>=2.6.3
22
24
  Provides-Extra: dev
23
25
  Requires-Dist: pre-commit>=4.5.1; extra == 'dev'
24
26
  Requires-Dist: pytest>=9.0.2; extra == 'dev'
@@ -0,0 +1,66 @@
1
+ dm_cli/__init__.py,sha256=9YY96-1GuXPdzOeSEWKmn9tAZ72Fc9sM8JlJufdz18w,46
2
+ dm_cli/cli.py,sha256=giAVltKTVUB3MuG8KQcoDTJYTJYcnfICJtAsRVcKOhg,7121
3
+ dm_cli/dmss.py,sha256=bA_jU9lIdY7HizK1DETZViufiWlcmYyFhBXLvFadyeE,5981
4
+ dm_cli/domain.py,sha256=BMBK-WOQwBvDBEr_AoywdXAUPXMHIKIZLnMHpkuzqPs,5650
5
+ dm_cli/enums.py,sha256=7PFPV1u5mexc-p6s8gG-wb7B5SmAORh5esmPQ-zxUtg,1236
6
+ dm_cli/import_entity.py,sha256=mAqVeZlD-uojG4EySPRkNr5biJhFmMe-ziasUiwujqg,6998
7
+ dm_cli/import_package.py,sha256=3dv0GNjRSI1MSFGeQW4_i1G1VaIvsrvfwvY4wKTFeRY,11559
8
+ dm_cli/package_tree_from_zip.py,sha256=bpr9BoRaYKaowaP-zDif6km7mYJLbbby0f0kZHH6DTc,4624
9
+ dm_cli/state.py,sha256=F2L_0bj0P840WDvCmqR0TCsko0AsVyjiOP8SVKocG7s,198
10
+ dm_cli/command_group/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
+ dm_cli/command_group/data_source.py,sha256=XnWMiJ-ScGfauFEeQ8IoftnasUg5lXYr-6D7jvdE6II,7671
12
+ dm_cli/command_group/entities.py,sha256=5HVX-8_Hy4wbeUCBult4LMyyAXhg8IG4hTu08gXOsuE,6240
13
+ dm_cli/dmss_api/__init__.py,sha256=YPL3TK0SH515dzVPW7zobuzH-cTo1IiMTMRuf6N5Pig,4607
14
+ dm_cli/dmss_api/api_client.py,sha256=cm6nf1CFBbNxsYypjaEvN3SdPD5mlt6z4IQyUvnVAzY,28750
15
+ dm_cli/dmss_api/api_response.py,sha256=eMxw1mpmJcoGZ3gs9z6jM4oYoZ10Gjk333s9sKxGv7s,652
16
+ dm_cli/dmss_api/configuration.py,sha256=19jfVTBvHAfOLvtJfod2ApojtpcruYoUYN7EQpKEzCw,21504
17
+ dm_cli/dmss_api/exceptions.py,sha256=vuHbzbssMJHdaa7UpJiSSeSn58dgj4ZHhlyaH1ixmwo,6480
18
+ dm_cli/dmss_api/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
+ dm_cli/dmss_api/rest.py,sha256=9Dm343pteceqPyLtGbevqQPcqzjgOpaLs2EI8gsfoJ4,11217
20
+ dm_cli/dmss_api/api/__init__.py,sha256=Ww_2WJ6AlWtxFMV8wzvoN6jMbGFla5J3ozWxzYlVZA0,975
21
+ dm_cli/dmss_api/api/access_control_api.py,sha256=QE6AUbEkhvK81X2EUhBG8XLXGt-A4KdgyBPZORgKO-c,25983
22
+ dm_cli/dmss_api/api/attribute_api.py,sha256=BWLgXQgkqZqC_Rw0f9J50iYFye1W5J3b6cmvvY95Cf4,13183
23
+ dm_cli/dmss_api/api/blob_api.py,sha256=nh5dUZeav-JtK0FfhOlr2r7Q6_vlgzg6E5jtq6X9XwU,26481
24
+ dm_cli/dmss_api/api/blueprint_api.py,sha256=l_De6Hdl_mBwy0Lu7dse9YJQ0wDMe1RnFaszqJ5tc_o,25308
25
+ dm_cli/dmss_api/api/datasource_api.py,sha256=FXVZ1TwIjJ00KvQsnoxi1SOaBj33psl2DL6-l5HBC64,36861
26
+ dm_cli/dmss_api/api/default_api.py,sha256=_xh6oIfR4i9TG7080dl3OrD0Wb4OfEga1AHsEO89Wy8,412495
27
+ dm_cli/dmss_api/api/document_api.py,sha256=ut2qdFhPeGbM-3_ykNcIbAPaCzLYr_joc8nx27h3pDI,93407
28
+ dm_cli/dmss_api/api/entity_api.py,sha256=2OWWGD7eXFM0DQf2D6N1WVuCcztPHchUPiwqZFcRsmg,39700
29
+ dm_cli/dmss_api/api/export_api.py,sha256=hY4kjOHRIOn0Ok6_ULtyU0l4539Ez9Ek4kUuQPyyYqA,25310
30
+ dm_cli/dmss_api/api/file_api.py,sha256=yvn3Gu2vrdD9adMfOHU7k4qjpnbZHGLdb6QlVR4IvlI,14612
31
+ dm_cli/dmss_api/api/health_check_api.py,sha256=-rTaXnjIRkW1Si_OTa_l_bXbpe4xjWnxBS755GOwr_4,11370
32
+ dm_cli/dmss_api/api/lookup_table_api.py,sha256=-nH7WQVMNYu2Gyt5zt7rMIzg_xEythCASGGKbHOovpk,37258
33
+ dm_cli/dmss_api/api/meta_api.py,sha256=eAC1r2Y6xBrsRrtml3ilWQpQ4nWYjnyU2UEyIs8wus4,13538
34
+ dm_cli/dmss_api/api/personal_access_token_api.py,sha256=iS6CKNRHS-747hK8989o55zU5-_eDWwpUGV4Zi1aALs,34959
35
+ dm_cli/dmss_api/api/search_api.py,sha256=xYWw-BHl_w-V5mudFVAHu9WdmAFIoj41hMRoLreezkc,16895
36
+ dm_cli/dmss_api/api/whoami_api.py,sha256=KQ0zuMVLerwDgGYJN5VXponhkuDIePr7TSaNDV7CJwo,12039
37
+ dm_cli/dmss_api/models/__init__.py,sha256=AKLq-oLWP2-NwwUjEgSQIvawiIg_AJcPBQvyY4wNgAw,1496
38
+ dm_cli/dmss_api/models/access_control_list.py,sha256=6SKARxbN_D0Dei6l5zE1inrSvlJFozkhmgYLihqAa2Q,3046
39
+ dm_cli/dmss_api/models/access_level.py,sha256=ZaH6igJJqkZsPL3e_3anLY7a_5xnEeV2Q1_oa8xeykU,716
40
+ dm_cli/dmss_api/models/data_source_information.py,sha256=fzOEWuEXph7ECL6CeD5PQfK4NHcLRe1rMPlmQ3Qon8c,3091
41
+ dm_cli/dmss_api/models/data_source_request.py,sha256=DI82fgSAMwbmAIzJHJ1C6Coy4RUrmq6MRkeo8TH85EI,3214
42
+ dm_cli/dmss_api/models/dependency.py,sha256=s7Ns5aM1CaI4w3w-ggOcQJQuiTZjeEY3R06gTRKjq2I,3114
43
+ dm_cli/dmss_api/models/entity.py,sha256=akPTBM1Fr7Ny0oGfHO1tL06msxs3fZ8ejbqXx3YnldY,3447
44
+ dm_cli/dmss_api/models/error_response.py,sha256=4szu-dkTXtxSYPaCmkJYeXU6wjOteF3Q0z4NLFCDdmo,3189
45
+ dm_cli/dmss_api/models/export_meta_response.py,sha256=S-66jV8qOW-ZCBhlY2jD4muzxNvMEN1cqJZcA_5oJNE,3306
46
+ dm_cli/dmss_api/models/get_attribute_response.py,sha256=TAieFVQrfuNxR5ilIItRD59GfpM7pG3Qo4JgoUcWu98,2560
47
+ dm_cli/dmss_api/models/get_blueprint_response.py,sha256=66zeRqsnmNeH9e_Xa5FKsff19TGeGXKrxJeXao5S5yw,2772
48
+ dm_cli/dmss_api/models/lookup.py,sha256=Rjo8K_i11YEyL6do0K4HZ3uYFeImggIY8Fr86pjDZsg,5584
49
+ dm_cli/dmss_api/models/pat_data.py,sha256=EC0H-JN1pxSSAGlvnvhaLy5_DonErI1aQyrgW0kmHxo,3257
50
+ dm_cli/dmss_api/models/recipe.py,sha256=pxbRPcv3gi8Ye7DdaNZZhiR2dXGsNxOttffwL7VjrBk,4774
51
+ dm_cli/dmss_api/models/recipe_attribute.py,sha256=UxMB8jHt6Xls5qW73THmFAJF-ntDYEGlCDsAjq21nms,4879
52
+ dm_cli/dmss_api/models/repository.py,sha256=RvKqpdIbb6C6WxYr7RpgtVQRH0Z2OBodKacK_rp6ykc,5626
53
+ dm_cli/dmss_api/models/storage_attribute.py,sha256=_l2_dPWOt8Ewm13kMuybrFneTH-JjXvulpnX9bMzCvI,3334
54
+ dm_cli/dmss_api/models/storage_data_types.py,sha256=CLRbZfQ1v-b5A7XCOp5XBUNDDB2x5tCctlZzeQPnw6U,785
55
+ dm_cli/dmss_api/models/storage_recipe.py,sha256=qlsiNuwM0VDt3yuImGNVZQ9cLzPZ3f6k-q18FFArZKo,3811
56
+ dm_cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
57
+ dm_cli/utils/file_structure.py,sha256=efIcl7kr7LGm9-2AKt6M6uXe0L8NHNBg0Icy1xO8rSM,1020
58
+ dm_cli/utils/reference.py,sha256=V11edpj8mJSCbXlWlv8HW2zO4AwHRqDnmwdxKv5_qpA,5593
59
+ dm_cli/utils/resolve_local_ids.py,sha256=I9kXVZTQgqr07eu4CghGQVU99MAEXCdn6hacVX0AYOY,5726
60
+ dm_cli/utils/utils.py,sha256=UEuW3ZfQ4cqNLXqyP-fFRiK1bgmaPdIzvl8Z566cx-s,7345
61
+ dm_cli/utils/zip.py,sha256=Zdq6Z978i_1iDAvbrqZwJNsw4QersVtK0HeqFiRagdA,2781
62
+ development_framework_cli-1.8.0.dist-info/METADATA,sha256=iEUXZdnBCU0KSBGzMay10r2OP9kMcdiMl0yVznl8Wvs,8234
63
+ development_framework_cli-1.8.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
64
+ development_framework_cli-1.8.0.dist-info/entry_points.txt,sha256=PAOXUTc_jhfY9Hfs7BNFzh4TlbrOGYU7HgTM-P7h074,38
65
+ development_framework_cli-1.8.0.dist-info/licenses/LICENSE,sha256=zefthAHZDOYIcHUXefkN_Fvuy4s-nYfoLX_dsycVDxc,1064
66
+ development_framework_cli-1.8.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: hatchling 1.29.0
2
+ Generator: hatchling 1.31.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
dm_cli/__init__.py CHANGED
@@ -1 +1 @@
1
- VERSION = "1.7.3" # x-release-please-version
1
+ VERSION = "1.8.0" # x-release-please-version
dm_cli/cli.py CHANGED
@@ -11,8 +11,9 @@ from typing_extensions import Annotated
11
11
 
12
12
  from dm_cli import VERSION
13
13
  from dm_cli.command_group.data_source import data_source_app, reset_data_source
14
- from dm_cli.command_group.entities import entities_app, import_entity
14
+ from dm_cli.command_group.entities import entities_app
15
15
  from dm_cli.dmss import dmss_api, dmss_exception_wrapper, export
16
+ from dm_cli.import_entity import import_folders_as_one_package
16
17
  from dm_cli.state import state
17
18
  from dm_cli.utils.file_structure import get_app_dir_structure, get_json_files_in_dir
18
19
  from dm_cli.utils.utils import (
@@ -74,7 +75,11 @@ def import_plugin_blueprints(
74
75
  """
75
76
  state.force = True
76
77
  dmss_exception_wrapper(
77
- import_entity, source=f"{path}/blueprints/", destination=f"system/Plugins/{Path(path).name}", validate=validate
78
+ import_folders_as_one_package,
79
+ source_path=Path(path) / "blueprints",
80
+ destination="system/Plugins",
81
+ package_name=Path(path).name,
82
+ validate=validate,
78
83
  )
79
84
 
80
85
 
@@ -3,6 +3,7 @@ from pathlib import Path
3
3
 
4
4
  import emoji
5
5
  import typer
6
+ import urllib3
6
7
  from rich import print
7
8
  from tenacity import (
8
9
  retry,
@@ -36,7 +37,7 @@ def import_data_source(
36
37
  wait=wait_random_exponential(multiplier=1, max=60),
37
38
  stop=stop_after_attempt(5),
38
39
  reraise=True,
39
- retry=retry_if_exception_type(ServiceException),
40
+ retry=retry_if_exception_type((ServiceException, urllib3.exceptions.HTTPError)),
40
41
  )
41
42
  def retry_wrapper():
42
43
  data_source_path = Path(path)
@@ -48,14 +49,14 @@ def import_data_source(
48
49
  # Read the data source definition
49
50
  with open(data_source_path) as file:
50
51
  document = json.load(file)
51
- existing_data_sources = dmss_exception_wrapper(dmss_api.data_source_get_all)
52
- if any(existing_document["name"] == document["name"] for existing_document in existing_data_sources):
52
+ existing_data_sources = dmss_api.data_source_get_all()
53
+ if any(existing_document.name == document["name"] for existing_document in existing_data_sources):
53
54
  print(f"WARNING: data source {document['name']} already exists. Updating existing data source.")
54
55
 
55
- dmss_exception_wrapper(dmss_api.data_source_save, document["name"], document)
56
+ dmss_api.data_source_save(document["name"], document)
56
57
  print(f"\tImported data source '{document['name']}' ✓")
57
58
 
58
- retry_wrapper()
59
+ dmss_exception_wrapper(retry_wrapper)
59
60
 
60
61
 
61
62
  @data_source_app.command("import-all", help="Import all datasources found in the directory given by 'path'")
@@ -1,5 +1,6 @@
1
1
  import json
2
2
  from pathlib import Path
3
+ from typing import List, Optional
3
4
 
4
5
  import typer
5
6
  from rich import print
@@ -70,24 +71,63 @@ def import_entity(
70
71
  return dmss_exception_wrapper(inner_import)
71
72
 
72
73
 
74
+ @entities_app.command("import-batch")
75
+ def import_batch(
76
+ data_directory: Annotated[
77
+ Path,
78
+ typer.Argument(
79
+ help="Path to a directory laid out as <dataSource>/<rootPackage>. Every root package found is imported into the data source it is located under."
80
+ ),
81
+ ],
82
+ exclude: Annotated[
83
+ Optional[List[str]],
84
+ typer.Option(help="Name of a data source to skip. Can be repeated."),
85
+ ] = None,
86
+ validate: Annotated[bool, typer.Option(help="if True, all entities uploaded will be validated.")] = True,
87
+ ) -> bool:
88
+ """
89
+ Import every root package found under <data_directory> into its data source.
90
+
91
+ This is equivalent to running 'entities import <data_directory>/<dataSource>/<rootPackage> <dataSource>'
92
+ once per root package, but does so in a single process and over a single reused HTTP connection.
93
+ """
94
+ if not data_directory.is_dir():
95
+ raise FileNotFoundError(f"The path '{data_directory}' is not a directory.")
96
+
97
+ def visible_entries(directory: Path) -> list[Path]:
98
+ # Hidden entries are skipped so that incidental files such as '.DS_Store'
99
+ # are not mistaken for packages.
100
+ return sorted(entry for entry in directory.iterdir() if not entry.name.startswith("."))
101
+
102
+ excluded = set(exclude or [])
103
+ for data_source_dir in visible_entries(data_directory):
104
+ if not data_source_dir.is_dir():
105
+ continue
106
+ if data_source_dir.name in excluded:
107
+ print(f"Skipping data source '{data_source_dir.name}'")
108
+ continue
109
+ for root_package in visible_entries(data_source_dir):
110
+ import_entity(str(root_package), data_source_dir.name, validate)
111
+ return True
112
+
113
+
73
114
  @entities_app.command("validate")
74
115
  def validate_entity(
75
- destination: Annotated[
76
- str,
116
+ destinations: Annotated[
117
+ List[str],
77
118
  typer.Argument(
78
- help="Address for the folder or file. Should be on the format <DataSource>/<rootPackage>/<subPackage>/<entity>"
119
+ help="Addresses for the folders or files to validate. Should be on the format <DataSource>/<rootPackage>/<subPackage>/<entity>"
79
120
  ),
80
121
  ],
81
122
  ) -> bool:
82
- """Recursively validate entity at remote target"""
83
- print(f"Validating entities recursively in: {destination}")
123
+ """Recursively validate one or more entities at remote targets"""
84
124
 
85
125
  @retry(
86
126
  wait=wait_random_exponential(multiplier=1, max=60),
87
127
  stop=stop_after_attempt(5),
88
128
  retry=retry_if_not_exception_type((ApplicationException, RuntimeError)),
89
129
  )
90
- def validation_error_wrapper():
130
+ def validation_error_wrapper(destination: str):
91
131
  try:
92
132
  dmss_api.validate_existing_entity(destination)
93
133
  except ApiException as e:
@@ -97,7 +137,11 @@ def validate_entity(
97
137
  raise typer.Exit(code=1)
98
138
  raise e
99
139
 
100
- dmss_exception_wrapper(validation_error_wrapper)
140
+ for destination in destinations:
141
+ print(f"Validating entities recursively in: {destination}")
142
+ # Validation is CPU bound on the DMSS side, so there is nothing to gain from validating targets
143
+ # concurrently; doing so only makes the blueprint caches thrash.
144
+ dmss_exception_wrapper(validation_error_wrapper, destination)
101
145
 
102
146
 
103
147
  @entities_app.command("delete")
dm_cli/dmss.py CHANGED
@@ -1,7 +1,7 @@
1
1
  import json
2
+ import re
2
3
  from typing import Any, Callable
3
4
 
4
- import requests
5
5
  import typer
6
6
  from rich.console import Console
7
7
  from rich.text import Text
@@ -14,12 +14,64 @@ from tenacity import (
14
14
 
15
15
  from dm_cli.dmss_api import ApiException
16
16
  from dm_cli.dmss_api.api.default_api import DefaultApi
17
+ from dm_cli.dmss_api.api_client import ApiClient
17
18
  from dm_cli.dmss_api.exceptions import NotFoundException, ServiceException
18
19
  from dm_cli.state import state
19
20
 
20
21
  console = Console()
21
22
 
22
- dmss_api = DefaultApi()
23
+ # The concrete types a request body loaded from JSON is built from.
24
+ _JSON_TYPES = frozenset({str, int, float, bool, type(None)})
25
+
26
+ # Generated response types that are built only from dicts, lists and untyped values, such as
27
+ # "object", "Dict[str, object]" and "Dict[str, Optional[object]]".
28
+ _PLAIN_JSON_RESPONSE_TYPE = re.compile(r"(?:Dict\[str, |List\[|Optional\[)*object\]*")
29
+
30
+
31
+ class JsonApiClient(ApiClient):
32
+ """An ApiClient that handles plain JSON without the generated type dispatch.
33
+
34
+ Most DMSS endpoints take and return documents: arbitrarily nested JSON that the generated client
35
+ describes as "object". It still walks every node of every document on the way in and on the way
36
+ out, which is both slow and, for responses, broken.
37
+
38
+ This lives here rather than in dm_cli/dmss_api, because that package is generated by
39
+ 'generate-dmss-api.sh' and any edit to it is lost on the next regeneration.
40
+ """
41
+
42
+ def sanitize_for_serialization(self, obj):
43
+ """Serialize a request body, dispatching on the concrete type first.
44
+
45
+ Request bodies are nested dicts loaded from JSON, and this recurses over every node of every
46
+ document, so it is called millions of times during a package import. The generated
47
+ implementation tests each value against abstract base classes, which is far more expensive
48
+ than an exact type lookup; dispatching on the concrete type first makes it ~3x faster.
49
+ Anything that is not plain JSON still falls through to the generated implementation.
50
+ """
51
+ obj_type = type(obj)
52
+ if obj_type in _JSON_TYPES:
53
+ return obj
54
+ if obj_type is dict:
55
+ return {key: self.sanitize_for_serialization(val) for key, val in obj.items()}
56
+ if obj_type is list:
57
+ return [self.sanitize_for_serialization(item) for item in obj]
58
+ return super().sanitize_for_serialization(obj)
59
+
60
+ def deserialize(self, response_text, response_type, content_type):
61
+ """Deserialize a response, returning plain JSON as-is.
62
+
63
+ The generated deserializer recurses through a type string such as
64
+ "Dict[str, Optional[object]]" only to hand every leaf to a function that returns it
65
+ unchanged, so collapsing the whole type to "object" is exact and skips the walk. It also
66
+ works around a generator bug: openapi-generator 7.24 emits "Optional[...]" but its own
67
+ deserializer cannot parse it, and raises AttributeError on every successful document_get.
68
+ """
69
+ if isinstance(response_type, str) and _PLAIN_JSON_RESPONSE_TYPE.fullmatch(response_type):
70
+ response_type = "object"
71
+ return super().deserialize(response_text, response_type, content_type)
72
+
73
+
74
+ dmss_api = DefaultApi(api_client=JsonApiClient())
23
75
 
24
76
 
25
77
  class ApplicationException(Exception):
@@ -66,6 +118,9 @@ def export(absolute_document_ref: str):
66
118
  The reason dmss_api cannot be used directly is that there were some issues with interpreting the JSON schema,
67
119
  which caused the export function in the generated DMSS api to not work properly.
68
120
  """
121
+ # Imported lazily; 'requests' is only needed by this command and costs ~200ms to import.
122
+ import requests
123
+
69
124
  headers = {"Access-Key": state.token}
70
125
 
71
126
  response = requests.get(f"{state.dmss_url}/api/export/{absolute_document_ref}", headers=headers) # nosec
@@ -1,27 +1,114 @@
1
+ # coding: utf-8
2
+
1
3
  # flake8: noqa
2
4
 
3
5
  """
4
6
  Data Modelling Storage Service
5
7
 
6
- API for basic data modelling interaction # noqa: E501
8
+ API for basic data modelling interaction
7
9
 
8
- The version of the OpenAPI document: 1.3.0
9
- Generated by: https://openapi-generator.tech
10
- """
10
+ The version of the OpenAPI document: 1.28.4
11
+ Generated by OpenAPI Generator (https://openapi-generator.tech)
12
+
13
+ Do not edit the class manually.
14
+ """ # noqa: E501
11
15
 
12
16
 
13
17
  __version__ = "1.0.0"
14
18
 
19
+ # Define package exports
20
+ __all__ = [
21
+ "AccessControlApi",
22
+ "AttributeApi",
23
+ "BlobApi",
24
+ "BlueprintApi",
25
+ "DatasourceApi",
26
+ "DefaultApi",
27
+ "DocumentApi",
28
+ "EntityApi",
29
+ "ExportApi",
30
+ "FileApi",
31
+ "HealthCheckApi",
32
+ "LookupTableApi",
33
+ "MetaApi",
34
+ "PersonalAccessTokenApi",
35
+ "SearchApi",
36
+ "WhoamiApi",
37
+ "ApiResponse",
38
+ "ApiClient",
39
+ "Configuration",
40
+ "OpenApiException",
41
+ "ApiTypeError",
42
+ "ApiValueError",
43
+ "ApiKeyError",
44
+ "ApiAttributeError",
45
+ "ApiException",
46
+ "AccessControlList",
47
+ "AccessLevel",
48
+ "DataSourceInformation",
49
+ "DataSourceRequest",
50
+ "Dependency",
51
+ "Entity",
52
+ "ErrorResponse",
53
+ "ExportMetaResponse",
54
+ "GetAttributeResponse",
55
+ "GetBlueprintResponse",
56
+ "Lookup",
57
+ "PATData",
58
+ "Recipe",
59
+ "RecipeAttribute",
60
+ "Repository",
61
+ "StorageAttribute",
62
+ "StorageDataTypes",
63
+ "StorageRecipe",
64
+ ]
65
+
66
+ # import apis into sdk package
67
+ from dm_cli.dmss_api.api.access_control_api import AccessControlApi as AccessControlApi
68
+ from dm_cli.dmss_api.api.attribute_api import AttributeApi as AttributeApi
69
+ from dm_cli.dmss_api.api.blob_api import BlobApi as BlobApi
70
+ from dm_cli.dmss_api.api.blueprint_api import BlueprintApi as BlueprintApi
71
+ from dm_cli.dmss_api.api.datasource_api import DatasourceApi as DatasourceApi
72
+ from dm_cli.dmss_api.api.default_api import DefaultApi as DefaultApi
73
+ from dm_cli.dmss_api.api.document_api import DocumentApi as DocumentApi
74
+ from dm_cli.dmss_api.api.entity_api import EntityApi as EntityApi
75
+ from dm_cli.dmss_api.api.export_api import ExportApi as ExportApi
76
+ from dm_cli.dmss_api.api.file_api import FileApi as FileApi
77
+ from dm_cli.dmss_api.api.health_check_api import HealthCheckApi as HealthCheckApi
78
+ from dm_cli.dmss_api.api.lookup_table_api import LookupTableApi as LookupTableApi
79
+ from dm_cli.dmss_api.api.meta_api import MetaApi as MetaApi
80
+ from dm_cli.dmss_api.api.personal_access_token_api import PersonalAccessTokenApi as PersonalAccessTokenApi
81
+ from dm_cli.dmss_api.api.search_api import SearchApi as SearchApi
82
+ from dm_cli.dmss_api.api.whoami_api import WhoamiApi as WhoamiApi
83
+
15
84
  # import ApiClient
16
- from dm_cli.dmss_api.api_client import ApiClient
17
-
18
- # import Configuration
19
- from dm_cli.dmss_api.configuration import Configuration
20
-
21
- # import exceptions
22
- from dm_cli.dmss_api.exceptions import OpenApiException
23
- from dm_cli.dmss_api.exceptions import ApiAttributeError
24
- from dm_cli.dmss_api.exceptions import ApiTypeError
25
- from dm_cli.dmss_api.exceptions import ApiValueError
26
- from dm_cli.dmss_api.exceptions import ApiKeyError
27
- from dm_cli.dmss_api.exceptions import ApiException
85
+ from dm_cli.dmss_api.api_response import ApiResponse as ApiResponse
86
+ from dm_cli.dmss_api.api_client import ApiClient as ApiClient
87
+ from dm_cli.dmss_api.configuration import Configuration as Configuration
88
+ from dm_cli.dmss_api.exceptions import OpenApiException as OpenApiException
89
+ from dm_cli.dmss_api.exceptions import ApiTypeError as ApiTypeError
90
+ from dm_cli.dmss_api.exceptions import ApiValueError as ApiValueError
91
+ from dm_cli.dmss_api.exceptions import ApiKeyError as ApiKeyError
92
+ from dm_cli.dmss_api.exceptions import ApiAttributeError as ApiAttributeError
93
+ from dm_cli.dmss_api.exceptions import ApiException as ApiException
94
+
95
+ # import models into sdk package
96
+ from dm_cli.dmss_api.models.access_control_list import AccessControlList as AccessControlList
97
+ from dm_cli.dmss_api.models.access_level import AccessLevel as AccessLevel
98
+ from dm_cli.dmss_api.models.data_source_information import DataSourceInformation as DataSourceInformation
99
+ from dm_cli.dmss_api.models.data_source_request import DataSourceRequest as DataSourceRequest
100
+ from dm_cli.dmss_api.models.dependency import Dependency as Dependency
101
+ from dm_cli.dmss_api.models.entity import Entity as Entity
102
+ from dm_cli.dmss_api.models.error_response import ErrorResponse as ErrorResponse
103
+ from dm_cli.dmss_api.models.export_meta_response import ExportMetaResponse as ExportMetaResponse
104
+ from dm_cli.dmss_api.models.get_attribute_response import GetAttributeResponse as GetAttributeResponse
105
+ from dm_cli.dmss_api.models.get_blueprint_response import GetBlueprintResponse as GetBlueprintResponse
106
+ from dm_cli.dmss_api.models.lookup import Lookup as Lookup
107
+ from dm_cli.dmss_api.models.pat_data import PATData as PATData
108
+ from dm_cli.dmss_api.models.recipe import Recipe as Recipe
109
+ from dm_cli.dmss_api.models.recipe_attribute import RecipeAttribute as RecipeAttribute
110
+ from dm_cli.dmss_api.models.repository import Repository as Repository
111
+ from dm_cli.dmss_api.models.storage_attribute import StorageAttribute as StorageAttribute
112
+ from dm_cli.dmss_api.models.storage_data_types import StorageDataTypes as StorageDataTypes
113
+ from dm_cli.dmss_api.models.storage_recipe import StorageRecipe as StorageRecipe
114
+
@@ -1,3 +1,20 @@
1
- # do not import all apis into this module because that uses a lot of memory and stack frames
2
- # if you need the ability to import all apis from one package, import them with
3
- # from dm_cli.dmss_api.apis import AccessControlApi
1
+ # flake8: noqa
2
+
3
+ # import apis into api package
4
+ from dm_cli.dmss_api.api.access_control_api import AccessControlApi
5
+ from dm_cli.dmss_api.api.attribute_api import AttributeApi
6
+ from dm_cli.dmss_api.api.blob_api import BlobApi
7
+ from dm_cli.dmss_api.api.blueprint_api import BlueprintApi
8
+ from dm_cli.dmss_api.api.datasource_api import DatasourceApi
9
+ from dm_cli.dmss_api.api.default_api import DefaultApi
10
+ from dm_cli.dmss_api.api.document_api import DocumentApi
11
+ from dm_cli.dmss_api.api.entity_api import EntityApi
12
+ from dm_cli.dmss_api.api.export_api import ExportApi
13
+ from dm_cli.dmss_api.api.file_api import FileApi
14
+ from dm_cli.dmss_api.api.health_check_api import HealthCheckApi
15
+ from dm_cli.dmss_api.api.lookup_table_api import LookupTableApi
16
+ from dm_cli.dmss_api.api.meta_api import MetaApi
17
+ from dm_cli.dmss_api.api.personal_access_token_api import PersonalAccessTokenApi
18
+ from dm_cli.dmss_api.api.search_api import SearchApi
19
+ from dm_cli.dmss_api.api.whoami_api import WhoamiApi
20
+