cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -3,7 +3,6 @@ from collections.abc import Collection, Hashable, Iterable, Sequence
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Literal, TypeAlias, cast
|
|
5
5
|
|
|
6
|
-
from cognite.client import CogniteClient
|
|
7
6
|
from cognite.client.data_classes._base import CogniteResource, CogniteResourceList
|
|
8
7
|
from cognite.client.data_classes.data_modeling import (
|
|
9
8
|
ContainerApplyList,
|
|
@@ -15,23 +14,14 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
15
14
|
)
|
|
16
15
|
from cognite.client.exceptions import CogniteAPIError
|
|
17
16
|
|
|
17
|
+
from cognite.neat._client import DataModelingLoader, NeatClient
|
|
18
|
+
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
18
19
|
from cognite.neat._issues import IssueList
|
|
19
20
|
from cognite.neat._issues.warnings import (
|
|
20
21
|
PrincipleOneModelOneSpaceWarning,
|
|
21
22
|
ResourceRetrievalWarning,
|
|
22
23
|
)
|
|
23
|
-
from cognite.neat._rules.models.dms import DMSRules
|
|
24
|
-
from cognite.neat._utils.cdf.loaders import (
|
|
25
|
-
ContainerLoader,
|
|
26
|
-
DataModelingLoader,
|
|
27
|
-
DataModelLoader,
|
|
28
|
-
RawDatabaseLoader,
|
|
29
|
-
RawTableLoader,
|
|
30
|
-
ResourceLoader,
|
|
31
|
-
SpaceLoader,
|
|
32
|
-
TransformationLoader,
|
|
33
|
-
ViewLoader,
|
|
34
|
-
)
|
|
24
|
+
from cognite.neat._rules.models.dms import DMSRules
|
|
35
25
|
from cognite.neat._utils.upload import UploadResult
|
|
36
26
|
|
|
37
27
|
from ._base import CDFExporter
|
|
@@ -119,14 +109,14 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
119
109
|
return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
|
|
120
110
|
|
|
121
111
|
def delete_from_cdf(
|
|
122
|
-
self, rules: DMSRules, client:
|
|
112
|
+
self, rules: DMSRules, client: NeatClient, dry_run: bool = False, skip_space: bool = False
|
|
123
113
|
) -> Iterable[UploadResult]:
|
|
124
|
-
to_export = self._prepare_exporters(rules
|
|
114
|
+
to_export = self._prepare_exporters(rules)
|
|
125
115
|
|
|
126
116
|
# we need to reverse order in which we are picking up the items to delete
|
|
127
117
|
# as they are sorted in the order of creation and we need to delete them in reverse order
|
|
128
118
|
for items, loader in reversed(to_export):
|
|
129
|
-
if skip_space and isinstance(
|
|
119
|
+
if skip_space and isinstance(items, SpaceApplyList):
|
|
130
120
|
continue
|
|
131
121
|
item_ids = loader.get_ids(items)
|
|
132
122
|
existing_items = loader.retrieve(item_ids)
|
|
@@ -166,9 +156,9 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
166
156
|
)
|
|
167
157
|
|
|
168
158
|
def export_to_cdf_iterable(
|
|
169
|
-
self, rules: DMSRules, client:
|
|
159
|
+
self, rules: DMSRules, client: NeatClient, dry_run: bool = False, fallback_one_by_one: bool = False
|
|
170
160
|
) -> Iterable[UploadResult]:
|
|
171
|
-
to_export = self._prepare_exporters(rules
|
|
161
|
+
to_export = self._prepare_exporters(rules)
|
|
172
162
|
|
|
173
163
|
result_by_name = {}
|
|
174
164
|
if self.existing_handling == "force":
|
|
@@ -176,17 +166,18 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
176
166
|
result_by_name[delete_result.name] = delete_result
|
|
177
167
|
|
|
178
168
|
redeploy_data_model = False
|
|
179
|
-
for items
|
|
169
|
+
for items in to_export:
|
|
180
170
|
# The conversion from DMS to GraphQL does not seem to be triggered even if the views
|
|
181
171
|
# are changed. This is a workaround to force the conversion.
|
|
182
|
-
is_redeploying =
|
|
172
|
+
is_redeploying = isinstance(items, DataModelApplyList) and redeploy_data_model
|
|
173
|
+
loader = client.loaders.get_loader(items)
|
|
183
174
|
|
|
184
175
|
to_create, to_delete, to_update, unchanged = self._categorize_items_for_upload(
|
|
185
176
|
loader, items, is_redeploying
|
|
186
177
|
)
|
|
187
178
|
|
|
188
179
|
issue_list = IssueList()
|
|
189
|
-
warning_list = self._validate(loader, items)
|
|
180
|
+
warning_list = self._validate(loader, items, client)
|
|
190
181
|
issue_list.extend(warning_list)
|
|
191
182
|
|
|
192
183
|
created: set[Hashable] = set()
|
|
@@ -206,6 +197,8 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
206
197
|
failed_changed.update(loader.get_id(item) for item in to_update)
|
|
207
198
|
else:
|
|
208
199
|
raise ValueError(f"Unsupported existing_handling {self.existing_handling}")
|
|
200
|
+
created.update(loader.get_id(item) for item in to_create)
|
|
201
|
+
deleted.update(loader.get_id(item) for item in to_delete)
|
|
209
202
|
else:
|
|
210
203
|
if to_delete:
|
|
211
204
|
try:
|
|
@@ -226,7 +219,7 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
226
219
|
else:
|
|
227
220
|
deleted.update(loader.get_id(item) for item in to_delete)
|
|
228
221
|
|
|
229
|
-
if isinstance(
|
|
222
|
+
if isinstance(items, DataModelApplyList):
|
|
230
223
|
to_create = loader.sort_by_dependencies(to_create)
|
|
231
224
|
|
|
232
225
|
try:
|
|
@@ -292,11 +285,11 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
292
285
|
issues=issue_list,
|
|
293
286
|
)
|
|
294
287
|
|
|
295
|
-
if
|
|
288
|
+
if isinstance(items, ViewApplyList) and (created or changed):
|
|
296
289
|
redeploy_data_model = True
|
|
297
290
|
|
|
298
291
|
def _categorize_items_for_upload(
|
|
299
|
-
self, loader:
|
|
292
|
+
self, loader: DataModelingLoader, items: Sequence[CogniteResource], is_redeploying
|
|
300
293
|
) -> tuple[list[CogniteResource], list[CogniteResource], list[CogniteResource], list[CogniteResource]]:
|
|
301
294
|
item_ids = loader.get_ids(items)
|
|
302
295
|
cdf_items = loader.retrieve(item_ids)
|
|
@@ -304,7 +297,7 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
304
297
|
to_create, to_update, unchanged, to_delete = [], [], [], []
|
|
305
298
|
for item in items:
|
|
306
299
|
if (
|
|
307
|
-
isinstance(
|
|
300
|
+
isinstance(items, DataModelApplyList)
|
|
308
301
|
and self.include_space is not None
|
|
309
302
|
and not loader.in_space(item, self.include_space)
|
|
310
303
|
):
|
|
@@ -322,28 +315,24 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
322
315
|
to_update.append(item)
|
|
323
316
|
return to_create, to_delete, to_update, unchanged
|
|
324
317
|
|
|
325
|
-
def _prepare_exporters(self, rules
|
|
318
|
+
def _prepare_exporters(self, rules: DMSRules) -> list[CogniteResourceList]:
|
|
326
319
|
schema = self.export(rules)
|
|
327
|
-
to_export: list[
|
|
320
|
+
to_export: list[CogniteResourceList] = []
|
|
328
321
|
if self.export_components.intersection({"all", "spaces"}):
|
|
329
|
-
to_export.append(
|
|
322
|
+
to_export.append(SpaceApplyList(schema.spaces.values()))
|
|
330
323
|
if self.export_components.intersection({"all", "containers"}):
|
|
331
|
-
to_export.append(
|
|
324
|
+
to_export.append(ContainerApplyList(schema.containers.values()))
|
|
332
325
|
if self.export_components.intersection({"all", "views"}):
|
|
333
|
-
to_export.append(
|
|
326
|
+
to_export.append(ViewApplyList(schema.views.values()))
|
|
334
327
|
if self.export_components.intersection({"all", "data_models"}):
|
|
335
|
-
to_export.append(
|
|
336
|
-
if isinstance(schema, PipelineSchema):
|
|
337
|
-
to_export.append((schema.databases, RawDatabaseLoader(client)))
|
|
338
|
-
to_export.append((schema.raw_tables, RawTableLoader(client)))
|
|
339
|
-
to_export.append((schema.transformations, TransformationLoader(client)))
|
|
328
|
+
to_export.append(DataModelApplyList([schema.data_model]))
|
|
340
329
|
return to_export
|
|
341
330
|
|
|
342
|
-
def _validate(self, loader:
|
|
331
|
+
def _validate(self, loader: DataModelingLoader, items: CogniteResourceList, client: NeatClient) -> IssueList:
|
|
343
332
|
issue_list = IssueList()
|
|
344
|
-
if isinstance(
|
|
333
|
+
if isinstance(items, DataModelApplyList):
|
|
345
334
|
models = cast(list[DataModelApply], items)
|
|
346
|
-
if other_models := self._exist_other_data_models(
|
|
335
|
+
if other_models := self._exist_other_data_models(client, models):
|
|
347
336
|
warning = PrincipleOneModelOneSpaceWarning(
|
|
348
337
|
f"There are multiple data models in the same space {models[0].space}. "
|
|
349
338
|
f"Other data models in the space are {other_models}.",
|
|
@@ -355,13 +344,13 @@ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
|
355
344
|
return issue_list
|
|
356
345
|
|
|
357
346
|
@classmethod
|
|
358
|
-
def _exist_other_data_models(cls,
|
|
347
|
+
def _exist_other_data_models(cls, client: NeatClient, models: list[DataModelApply]) -> list[DataModelId]:
|
|
359
348
|
if not models:
|
|
360
349
|
return []
|
|
361
350
|
space = models[0].space
|
|
362
351
|
external_id = models[0].external_id
|
|
363
352
|
try:
|
|
364
|
-
data_models =
|
|
353
|
+
data_models = client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
|
|
365
354
|
except CogniteAPIError as e:
|
|
366
355
|
warnings.warn(ResourceRetrievalWarning(frozenset({space}), "space", str(e)), stacklevel=2)
|
|
367
356
|
return []
|
|
@@ -13,17 +13,13 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
|
13
13
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
14
14
|
from rdflib import Namespace
|
|
15
15
|
|
|
16
|
-
from cognite.neat._constants import COGNITE_MODELS
|
|
17
16
|
from cognite.neat._rules._shared import VerifiedRules
|
|
18
17
|
from cognite.neat._rules.models import (
|
|
19
|
-
DataModelType,
|
|
20
18
|
ExtensionCategory,
|
|
21
19
|
SchemaCompleteness,
|
|
22
20
|
SheetRow,
|
|
23
21
|
)
|
|
24
22
|
from cognite.neat._rules.models.dms import DMSMetadata
|
|
25
|
-
from cognite.neat._rules.models.dms._rules import DMSRules
|
|
26
|
-
from cognite.neat._rules.models.domain import DomainMetadata
|
|
27
23
|
from cognite.neat._rules.models.information import InformationMetadata
|
|
28
24
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
29
25
|
|
|
@@ -40,23 +36,13 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
40
36
|
on the different styles.
|
|
41
37
|
output_role: The role to use for the exported spreadsheet. If provided, the rules will be converted to
|
|
42
38
|
this role formate before being written to excel. If not provided, the role from the rules will be used.
|
|
43
|
-
dump_as: This determines how the rules are written to the Excel file. An Excel file has up to three sets of
|
|
44
|
-
sheets: user, last, and reference. The user sheets are used for inputting rules from a user. The last sheets
|
|
45
|
-
are used for the last version of the same model as the user, while the reference sheets are used for
|
|
46
|
-
the model the user is building on. The options are:
|
|
47
|
-
* "user": The rules are written to the user sheets. This is used when you want to modify the rules
|
|
48
|
-
directly and potentially change the model. This is useful when you have imported the data model
|
|
49
|
-
from outside CDF and you want to modify it before you write it to CDF.
|
|
50
|
-
* "last": The rules are written to the last sheets. This is used when you want to extend the rules,
|
|
51
|
-
but have validation that you are not breaking the existing model. This is used when you want to
|
|
52
|
-
change a model that has already been published to CDF and that model is in production.
|
|
53
|
-
* "reference": The rules are written to the reference sheets. This is typically used when you want to build
|
|
54
|
-
a new solution on top of an enterprise model.
|
|
55
39
|
new_model_id: The new model ID to use for the exported spreadsheet. This is only applicable if the input
|
|
56
40
|
rules have 'is_reference' set. If provided, the model ID will be used to automatically create the
|
|
57
41
|
new metadata sheet in the Excel file. The model id is expected to be a tuple of (prefix, title)
|
|
58
42
|
(space, external_id) for InformationRules and DMSRules respectively.
|
|
59
43
|
|
|
44
|
+
sheet_prefix: The prefix to use for the sheet names in the Excel file. Defaults to an empty string.
|
|
45
|
+
|
|
60
46
|
The following styles are available:
|
|
61
47
|
|
|
62
48
|
- "none": No styling is applied.
|
|
@@ -80,16 +66,17 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
80
66
|
dump_options = get_args(DumpOptions)
|
|
81
67
|
|
|
82
68
|
def __init__(
|
|
83
|
-
self,
|
|
69
|
+
self,
|
|
70
|
+
styling: Style = "default",
|
|
71
|
+
new_model_id: tuple[str, str] | None = None,
|
|
72
|
+
sheet_prefix: str | None = None,
|
|
84
73
|
):
|
|
74
|
+
self.sheet_prefix = sheet_prefix or ""
|
|
85
75
|
if styling not in self.style_options:
|
|
86
76
|
raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
|
|
87
|
-
if dump_as not in self.dump_options:
|
|
88
|
-
raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
|
|
89
77
|
self.styling = styling
|
|
90
78
|
self._styling_level = self.style_options.index(styling)
|
|
91
79
|
self.new_model_id = new_model_id
|
|
92
|
-
self.dump_as = dump_as
|
|
93
80
|
|
|
94
81
|
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
95
82
|
"""Exports transformation rules to excel file."""
|
|
@@ -105,43 +92,10 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
105
92
|
# Remove default sheet named "Sheet"
|
|
106
93
|
workbook.remove(workbook["Sheet"])
|
|
107
94
|
|
|
108
|
-
dumped_user_rules: dict[str, Any]
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
action = {"last": "update", "reference": "create"}[self.dump_as]
|
|
113
|
-
metadata_creator = _MetadataCreator(action, self.new_model_id) # type: ignore[arg-type]
|
|
114
|
-
|
|
115
|
-
dumped_user_rules = {
|
|
116
|
-
"Metadata": metadata_creator.create(rules.metadata),
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
if self.dump_as == "last":
|
|
120
|
-
dumped_last_rules = rules.dump(by_alias=True)
|
|
121
|
-
if rules.reference:
|
|
122
|
-
dumped_reference_rules = rules.reference.dump(by_alias=True, as_reference=True)
|
|
123
|
-
elif self.dump_as == "reference":
|
|
124
|
-
dumped_reference_rules = rules.dump(by_alias=True, as_reference=True)
|
|
125
|
-
else:
|
|
126
|
-
dumped_user_rules = rules.dump(by_alias=True)
|
|
127
|
-
if rules.last:
|
|
128
|
-
dumped_last_rules = rules.last.dump(by_alias=True)
|
|
129
|
-
if rules.reference:
|
|
130
|
-
dumped_reference_rules = rules.reference.dump(by_alias=True, as_reference=True)
|
|
131
|
-
|
|
132
|
-
self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"])
|
|
133
|
-
self._write_sheets(workbook, dumped_user_rules, rules)
|
|
134
|
-
if dumped_last_rules:
|
|
135
|
-
self._write_sheets(workbook, dumped_last_rules, rules, sheet_prefix="Last")
|
|
136
|
-
self._write_metadata_sheet(workbook, dumped_last_rules["Metadata"], sheet_prefix="Last")
|
|
137
|
-
|
|
138
|
-
if dumped_reference_rules:
|
|
139
|
-
if isinstance(rules.reference, DMSRules):
|
|
140
|
-
sheet_prefix = "CDM" if rules.reference.metadata.as_data_model_id() in COGNITE_MODELS else "Ref"
|
|
141
|
-
else:
|
|
142
|
-
sheet_prefix = "Ref"
|
|
143
|
-
self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix=sheet_prefix)
|
|
144
|
-
self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix=sheet_prefix)
|
|
95
|
+
dumped_user_rules: dict[str, Any] = rules.dump(by_alias=True)
|
|
96
|
+
|
|
97
|
+
self._write_metadata_sheet(workbook, dumped_user_rules["Metadata"], sheet_prefix=self.sheet_prefix)
|
|
98
|
+
self._write_sheets(workbook, dumped_user_rules, rules, sheet_prefix=self.sheet_prefix)
|
|
145
99
|
|
|
146
100
|
if isinstance(rules, InformationRules) and rules.prefixes:
|
|
147
101
|
self._write_prefixes_sheet(workbook, rules.prefixes)
|
|
@@ -274,9 +228,9 @@ class _MetadataCreator:
|
|
|
274
228
|
new_model_id: tuple[str, str] | None = None,
|
|
275
229
|
):
|
|
276
230
|
self.action = action
|
|
277
|
-
self.new_model_id = new_model_id or ("
|
|
231
|
+
self.new_model_id = new_model_id or ("YOUR_SPACE", "YOUR_EXTERNAL_ID")
|
|
278
232
|
|
|
279
|
-
def create(self, metadata:
|
|
233
|
+
def create(self, metadata: InformationMetadata | DMSMetadata) -> dict[str, Any]:
|
|
280
234
|
now = datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None)
|
|
281
235
|
if self.action == "update":
|
|
282
236
|
output = json.loads(metadata.model_dump_json(by_alias=True))
|
|
@@ -290,13 +244,6 @@ class _MetadataCreator:
|
|
|
290
244
|
output["creator"] = self.creator_name
|
|
291
245
|
return output
|
|
292
246
|
|
|
293
|
-
# Action "create"
|
|
294
|
-
if isinstance(metadata, DomainMetadata):
|
|
295
|
-
output = {field_alias: None for field_alias in metadata.model_dump(by_alias=True).keys()}
|
|
296
|
-
output["role"] = metadata.role.value
|
|
297
|
-
output["creator"] = self.creator_name
|
|
298
|
-
return output
|
|
299
|
-
|
|
300
247
|
new_metadata = self._create_new_info(now)
|
|
301
248
|
if isinstance(metadata, DMSMetadata):
|
|
302
249
|
from cognite.neat._rules.transformers._converters import _InformationRulesConverter
|
|
@@ -314,13 +261,9 @@ class _MetadataCreator:
|
|
|
314
261
|
return created
|
|
315
262
|
|
|
316
263
|
def _create_new_info(self, now: datetime) -> InformationMetadata:
|
|
317
|
-
prefix = self.new_model_id[0]
|
|
318
264
|
return InformationMetadata(
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
extension=ExtensionCategory.addition,
|
|
322
|
-
prefix=prefix,
|
|
323
|
-
namespace=f"http://purl.org/neat/{prefix}/", # type: ignore[arg-type]
|
|
265
|
+
space=self.new_model_id[0],
|
|
266
|
+
external_id=self.new_model_id[1],
|
|
324
267
|
description=None,
|
|
325
268
|
version="1",
|
|
326
269
|
created=now,
|
|
@@ -247,9 +247,9 @@ class OWLClass(OntologyModel):
|
|
|
247
247
|
|
|
248
248
|
@classmethod
|
|
249
249
|
def from_class(cls, definition: InformationClass, namespace: Namespace, prefixes: dict) -> Self:
|
|
250
|
-
if definition.
|
|
250
|
+
if definition.implements and isinstance(definition.implements, list):
|
|
251
251
|
sub_class_of = []
|
|
252
|
-
for parent_class in definition.
|
|
252
|
+
for parent_class in definition.implements:
|
|
253
253
|
try:
|
|
254
254
|
sub_class_of.append(prefixes[str(parent_class.prefix)][str(parent_class.suffix)])
|
|
255
255
|
except KeyError:
|
|
@@ -524,8 +524,8 @@ class SHACLNodeShape(OntologyModel):
|
|
|
524
524
|
def from_rules(
|
|
525
525
|
cls, class_definition: InformationClass, property_definitions: list[InformationProperty], namespace: Namespace
|
|
526
526
|
) -> "SHACLNodeShape":
|
|
527
|
-
if class_definition.
|
|
528
|
-
parent = [namespace[str(parent.suffix) + "Shape"] for parent in class_definition.
|
|
527
|
+
if class_definition.implements:
|
|
528
|
+
parent = [namespace[str(parent.suffix) + "Shape"] for parent in class_definition.implements]
|
|
529
529
|
else:
|
|
530
530
|
parent = None
|
|
531
531
|
return cls(
|
|
@@ -6,7 +6,6 @@ from datetime import datetime
|
|
|
6
6
|
from typing import Any, Generic, Literal
|
|
7
7
|
|
|
8
8
|
from pydantic import ValidationError
|
|
9
|
-
from rdflib import Namespace
|
|
10
9
|
|
|
11
10
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
12
11
|
from cognite.neat._issues import IssueList, NeatError, NeatWarning
|
|
@@ -33,11 +32,11 @@ class BaseImporter(ABC, Generic[T_InputRules]):
|
|
|
33
32
|
creator = getpass.getuser()
|
|
34
33
|
|
|
35
34
|
return {
|
|
36
|
-
"prefix": "neat",
|
|
37
35
|
"schema": "partial",
|
|
38
|
-
"
|
|
36
|
+
"space": "neat",
|
|
37
|
+
"external_id": "NeatImportedDataModel",
|
|
39
38
|
"version": "0.1.0",
|
|
40
|
-
"
|
|
39
|
+
"name": "Neat Imported Data Model",
|
|
41
40
|
"created": datetime.now().replace(microsecond=0).isoformat(),
|
|
42
41
|
"updated": datetime.now().replace(microsecond=0).isoformat(),
|
|
43
42
|
"creator": creator,
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
|
-
from collections.abc import Collection, Sequence
|
|
2
|
+
from collections.abc import Collection, Iterable, Sequence
|
|
3
3
|
from datetime import datetime, timezone
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Literal, cast
|
|
6
6
|
|
|
7
|
-
from cognite.client import CogniteClient
|
|
8
7
|
from cognite.client import data_modeling as dm
|
|
9
8
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
10
9
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
|
|
@@ -19,6 +18,7 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
19
18
|
)
|
|
20
19
|
from cognite.client.utils import ms_to_datetime
|
|
21
20
|
|
|
21
|
+
from cognite.neat._client import NeatClient
|
|
22
22
|
from cognite.neat._issues import IssueList, NeatIssue
|
|
23
23
|
from cognite.neat._issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
|
|
24
24
|
from cognite.neat._issues.warnings import (
|
|
@@ -30,10 +30,8 @@ from cognite.neat._issues.warnings import (
|
|
|
30
30
|
from cognite.neat._rules._shared import ReadRules
|
|
31
31
|
from cognite.neat._rules.importers._base import BaseImporter, _handle_issues
|
|
32
32
|
from cognite.neat._rules.models import (
|
|
33
|
-
DataModelType,
|
|
34
33
|
DMSInputRules,
|
|
35
34
|
DMSSchema,
|
|
36
|
-
SchemaCompleteness,
|
|
37
35
|
)
|
|
38
36
|
from cognite.neat._rules.models.data_types import DataType, Enum
|
|
39
37
|
from cognite.neat._rules.models.dms import (
|
|
@@ -86,10 +84,18 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
86
84
|
self._all_containers_by_id.update(schema.reference.containers.items())
|
|
87
85
|
self._all_views_by_id.update(schema.reference.views.items())
|
|
88
86
|
|
|
87
|
+
def update_referenced_containers(self, containers: Iterable[dm.ContainerApply]) -> None:
|
|
88
|
+
"""Update the referenced containers. This is useful to add Cognite containers identified after the root schema
|
|
89
|
+
is read"""
|
|
90
|
+
for container in containers:
|
|
91
|
+
if container.as_id() in self._all_containers_by_id:
|
|
92
|
+
continue
|
|
93
|
+
self._all_containers_by_id[container.as_id()] = container
|
|
94
|
+
|
|
89
95
|
@classmethod
|
|
90
96
|
def from_data_model_id(
|
|
91
97
|
cls,
|
|
92
|
-
client:
|
|
98
|
+
client: NeatClient,
|
|
93
99
|
data_model_id: DataModelIdentifier,
|
|
94
100
|
reference_model_id: DataModelIdentifier | None = None,
|
|
95
101
|
) -> "DMSImporter":
|
|
@@ -141,12 +147,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
141
147
|
|
|
142
148
|
issue_list = IssueList()
|
|
143
149
|
with _handle_issues(issue_list) as result:
|
|
144
|
-
schema = DMSSchema.from_data_model(client, user_model, ref_model)
|
|
150
|
+
schema = DMSSchema.from_data_model(NeatClient(client), user_model, ref_model)
|
|
145
151
|
|
|
146
152
|
if result.result == "failure" or issue_list.has_errors:
|
|
147
153
|
return cls(DMSSchema(), issue_list)
|
|
148
154
|
|
|
149
|
-
metadata = cls._create_metadata_from_model(user_model
|
|
155
|
+
metadata = cls._create_metadata_from_model(user_model)
|
|
150
156
|
ref_metadata = cls._create_metadata_from_model(ref_model) if ref_model else None
|
|
151
157
|
|
|
152
158
|
return cls(schema, issue_list, metadata, ref_metadata)
|
|
@@ -168,7 +174,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
168
174
|
def _create_metadata_from_model(
|
|
169
175
|
cls,
|
|
170
176
|
model: dm.DataModel[dm.View] | dm.DataModelApply,
|
|
171
|
-
has_reference: bool = False,
|
|
172
177
|
) -> DMSInputMetadata:
|
|
173
178
|
description, creator = DMSInputMetadata._get_description_and_creator(model.description)
|
|
174
179
|
|
|
@@ -180,9 +185,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
180
185
|
created = now
|
|
181
186
|
updated = now
|
|
182
187
|
return DMSInputMetadata(
|
|
183
|
-
schema_="complete",
|
|
184
|
-
data_model_type="solution" if has_reference else "enterprise",
|
|
185
|
-
extension="addition",
|
|
186
188
|
space=model.space,
|
|
187
189
|
external_id=model.external_id,
|
|
188
190
|
name=model.name or model.external_id,
|
|
@@ -226,28 +228,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
226
228
|
|
|
227
229
|
model = self.root_schema.data_model
|
|
228
230
|
|
|
229
|
-
schema_completeness = SchemaCompleteness.complete
|
|
230
|
-
data_model_type = DataModelType.enterprise
|
|
231
|
-
reference: DMSInputRules | None = None
|
|
232
|
-
if (ref_schema := self.root_schema.reference) and (ref_model := ref_schema.data_model):
|
|
233
|
-
# Reference should always be an enterprise model.
|
|
234
|
-
reference = self._create_rule_components(
|
|
235
|
-
ref_model,
|
|
236
|
-
ref_schema,
|
|
237
|
-
self.ref_metadata or self._create_default_metadata(list(ref_schema.views.values()), is_ref=True),
|
|
238
|
-
DataModelType.enterprise,
|
|
239
|
-
)
|
|
240
|
-
data_model_type = DataModelType.solution
|
|
241
|
-
|
|
242
231
|
user_rules = self._create_rule_components(
|
|
243
232
|
model,
|
|
244
233
|
self.root_schema,
|
|
245
234
|
self.metadata,
|
|
246
|
-
data_model_type,
|
|
247
|
-
schema_completeness,
|
|
248
|
-
has_reference=reference is not None,
|
|
249
235
|
)
|
|
250
|
-
|
|
236
|
+
|
|
251
237
|
self._end = datetime.now(timezone.utc)
|
|
252
238
|
|
|
253
239
|
return ReadRules(user_rules, self.issue_list, {})
|
|
@@ -257,9 +243,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
257
243
|
data_model: dm.DataModelApply,
|
|
258
244
|
schema: DMSSchema,
|
|
259
245
|
metadata: DMSInputMetadata | None = None,
|
|
260
|
-
data_model_type: DataModelType | None = None,
|
|
261
|
-
schema_completeness: SchemaCompleteness | None = None,
|
|
262
|
-
has_reference: bool = False,
|
|
263
246
|
) -> DMSInputRules:
|
|
264
247
|
properties: list[DMSInputProperty] = []
|
|
265
248
|
for view_id, view in schema.views.items():
|
|
@@ -274,11 +257,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
274
257
|
view.as_id() if isinstance(view, dm.View | dm.ViewApply) else view for view in data_model.views or []
|
|
275
258
|
}
|
|
276
259
|
|
|
277
|
-
metadata = metadata or DMSInputMetadata.from_data_model(data_model
|
|
278
|
-
if data_model_type is not None:
|
|
279
|
-
metadata.data_model_type = str(data_model_type) # type: ignore[assignment]
|
|
280
|
-
if schema_completeness is not None:
|
|
281
|
-
metadata.schema_ = str(schema_completeness) # type: ignore[assignment]
|
|
260
|
+
metadata = metadata or DMSInputMetadata.from_data_model(data_model)
|
|
282
261
|
|
|
283
262
|
enum = self._create_enum_collections(schema.containers.values())
|
|
284
263
|
|
|
@@ -301,9 +280,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
301
280
|
now = datetime.now().replace(microsecond=0)
|
|
302
281
|
space = Counter(view.space for view in views).most_common(1)[0][0]
|
|
303
282
|
return DMSInputMetadata(
|
|
304
|
-
schema_="complete",
|
|
305
|
-
extension="addition",
|
|
306
|
-
data_model_type="enterprise" if is_ref else "solution",
|
|
307
283
|
space=space,
|
|
308
284
|
external_id="Unknown",
|
|
309
285
|
version="0.1.0",
|
|
@@ -351,8 +327,6 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
351
327
|
return None
|
|
352
328
|
|
|
353
329
|
return DMSInputProperty(
|
|
354
|
-
class_=str(class_entity),
|
|
355
|
-
property_=prop_id,
|
|
356
330
|
description=prop.description,
|
|
357
331
|
name=prop.name,
|
|
358
332
|
connection=self._get_connection_type(prop),
|
|
@@ -361,10 +335,12 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
361
335
|
nullable=self._get_nullable(prop),
|
|
362
336
|
immutable=self._get_immutable(prop),
|
|
363
337
|
default=self._get_default(prop),
|
|
364
|
-
container=
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
container_property=
|
|
338
|
+
container=(
|
|
339
|
+
str(ContainerEntity.from_id(prop.container)) if isinstance(prop, dm.MappedPropertyApply) else None
|
|
340
|
+
),
|
|
341
|
+
container_property=(
|
|
342
|
+
prop.container_property_identifier if isinstance(prop, dm.MappedPropertyApply) else None
|
|
343
|
+
),
|
|
368
344
|
view=str(view_entity),
|
|
369
345
|
view_property=prop_id,
|
|
370
346
|
index=self._get_index(prop, prop_id),
|
|
@@ -99,8 +99,7 @@ class _DTDLConverter:
|
|
|
99
99
|
class_=item.id_.as_class_id(),
|
|
100
100
|
name=item.display_name,
|
|
101
101
|
description=item.description,
|
|
102
|
-
|
|
103
|
-
parent=[parent.as_class_id() for parent in item.extends or []] or None,
|
|
102
|
+
implements=[parent.as_class_id() for parent in item.extends or []] or None,
|
|
104
103
|
)
|
|
105
104
|
self.classes.append(class_)
|
|
106
105
|
for sub_item_or_id in item.contents or []:
|
|
@@ -135,7 +134,6 @@ class _DTDLConverter:
|
|
|
135
134
|
property_=item.name,
|
|
136
135
|
name=item.display_name,
|
|
137
136
|
description=item.description,
|
|
138
|
-
comment=item.comment,
|
|
139
137
|
value_type=value_type,
|
|
140
138
|
min_count=min_count,
|
|
141
139
|
max_count=max_count,
|
|
@@ -181,7 +179,6 @@ class _DTDLConverter:
|
|
|
181
179
|
property_=item.name,
|
|
182
180
|
name=item.display_name,
|
|
183
181
|
description=item.description,
|
|
184
|
-
comment=item.comment,
|
|
185
182
|
value_type=value_type,
|
|
186
183
|
min_count=0,
|
|
187
184
|
max_count=1,
|
|
@@ -201,7 +198,6 @@ class _DTDLConverter:
|
|
|
201
198
|
property_=item.name,
|
|
202
199
|
name=item.display_name,
|
|
203
200
|
description=item.description,
|
|
204
|
-
comment=item.comment,
|
|
205
201
|
value_type=value_type,
|
|
206
202
|
min_count=0,
|
|
207
203
|
max_count=1,
|
|
@@ -233,7 +229,6 @@ class _DTDLConverter:
|
|
|
233
229
|
description=item.description,
|
|
234
230
|
min_count=item.min_multiplicity or 0,
|
|
235
231
|
max_count=item.max_multiplicity or 1,
|
|
236
|
-
comment=item.comment,
|
|
237
232
|
value_type=value_type,
|
|
238
233
|
)
|
|
239
234
|
self.properties.append(prop)
|
|
@@ -255,7 +250,6 @@ class _DTDLConverter:
|
|
|
255
250
|
class_=item.id_.as_class_id(),
|
|
256
251
|
name=item.display_name,
|
|
257
252
|
description=item.description,
|
|
258
|
-
comment=item.comment,
|
|
259
253
|
)
|
|
260
254
|
self.classes.append(class_)
|
|
261
255
|
|
|
@@ -17,7 +17,7 @@ from cognite.neat._rules._shared import ReadRules
|
|
|
17
17
|
from cognite.neat._rules.importers._base import BaseImporter
|
|
18
18
|
from cognite.neat._rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
|
|
19
19
|
from cognite.neat._rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
20
|
-
from cognite.neat._rules.models import InformationInputRules
|
|
20
|
+
from cognite.neat._rules.models import InformationInputRules
|
|
21
21
|
from cognite.neat._rules.models.information import InformationInputMetadata
|
|
22
22
|
from cognite.neat._utils.text import humanize_collection, to_pascal
|
|
23
23
|
|
|
@@ -31,7 +31,7 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
|
31
31
|
|
|
32
32
|
Args:
|
|
33
33
|
items (Sequence[DTDLBase]): A sequence of DTDLBase objects.
|
|
34
|
-
|
|
34
|
+
name (str, optional): Name of the data model. Defaults to None.
|
|
35
35
|
read_issues (list[ValidationIssue], optional): A list of issues that occurred during reading. Defaults to None.
|
|
36
36
|
schema (SchemaCompleteness, optional): Schema completeness. Defaults to SchemaCompleteness.partial.
|
|
37
37
|
|
|
@@ -40,14 +40,12 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
|
40
40
|
def __init__(
|
|
41
41
|
self,
|
|
42
42
|
items: Sequence[DTDLBase],
|
|
43
|
-
|
|
43
|
+
name: str | None = None,
|
|
44
44
|
read_issues: list[NeatIssue] | None = None,
|
|
45
|
-
schema: SchemaCompleteness = SchemaCompleteness.partial,
|
|
46
45
|
) -> None:
|
|
47
46
|
self._items = items
|
|
48
|
-
self.
|
|
47
|
+
self.name = name
|
|
49
48
|
self._read_issues = IssueList(read_issues)
|
|
50
|
-
self._schema_completeness = schema
|
|
51
49
|
|
|
52
50
|
@classmethod
|
|
53
51
|
def _from_file_content(cls, file_content: str, filepath: Path) -> Iterable[DTDLBase | NeatIssue]:
|
|
@@ -130,17 +128,16 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
|
130
128
|
converter.convert(self._items)
|
|
131
129
|
|
|
132
130
|
metadata = self._default_metadata()
|
|
133
|
-
metadata["schema"] = self._schema_completeness.value
|
|
134
131
|
|
|
135
|
-
if self.
|
|
136
|
-
metadata["
|
|
132
|
+
if self.name:
|
|
133
|
+
metadata["name"] = to_pascal(self.name)
|
|
137
134
|
try:
|
|
138
135
|
most_common_prefix = converter.get_most_common_prefix()
|
|
139
136
|
except ValueError:
|
|
140
137
|
# No prefixes are defined so we just use the default prefix...
|
|
141
138
|
...
|
|
142
139
|
else:
|
|
143
|
-
metadata["
|
|
140
|
+
metadata["space"] = most_common_prefix
|
|
144
141
|
|
|
145
142
|
rules = InformationInputRules(
|
|
146
143
|
metadata=InformationInputMetadata.load(metadata),
|