cognite-neat 0.121.0__py3-none-any.whl → 0.121.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/core/_client/_api/statistics.py +91 -0
- cognite/neat/core/_client/_api_client.py +2 -0
- cognite/neat/core/_client/data_classes/statistics.py +125 -0
- cognite/neat/core/_client/testing.py +4 -0
- cognite/neat/core/_constants.py +6 -7
- cognite/neat/core/{_rules → _data_model}/_constants.py +25 -18
- cognite/neat/core/_data_model/_shared.py +59 -0
- cognite/neat/core/_data_model/analysis/__init__.py +3 -0
- cognite/neat/core/{_rules → _data_model}/analysis/_base.py +202 -195
- cognite/neat/core/{_rules → _data_model}/catalog/__init__.py +1 -1
- cognite/neat/core/{_rules → _data_model}/exporters/__init__.py +5 -5
- cognite/neat/core/{_rules → _data_model}/exporters/_base.py +10 -8
- cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py} +22 -18
- cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py} +61 -56
- cognite/neat/core/{_rules/exporters/_rules2instance_template.py → _data_model/exporters/_data_model2instance_template.py} +11 -9
- cognite/neat/core/{_rules/exporters/_rules2ontology.py → _data_model/exporters/_data_model2ontology.py} +64 -61
- cognite/neat/core/{_rules/exporters/_rules2yaml.py → _data_model/exporters/_data_model2yaml.py} +21 -18
- cognite/neat/core/{_rules → _data_model}/importers/__init__.py +6 -8
- cognite/neat/core/{_rules → _data_model}/importers/_base.py +8 -6
- cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
- cognite/neat/core/{_rules/importers/_yaml2rules.py → _data_model/importers/_dict2data_model.py} +41 -21
- cognite/neat/core/{_rules/importers/_dms2rules.py → _data_model/importers/_dms2data_model.py} +79 -66
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_converter.py +41 -41
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_importer.py +16 -16
- cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/spec.py +3 -3
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_base.py +18 -16
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_imf2rules.py +17 -17
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_inference2rules.py +50 -50
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_owl2rules.py +14 -14
- cognite/neat/core/{_rules → _data_model}/importers/_rdf/_shared.py +25 -25
- cognite/neat/core/{_rules/importers/_spreadsheet2rules.py → _data_model/importers/_spreadsheet2data_model.py} +69 -38
- cognite/neat/core/_data_model/models/__init__.py +36 -0
- cognite/neat/core/{_rules/models/_base_input.py → _data_model/models/_base_unverified.py} +12 -12
- cognite/neat/core/{_rules/models/_base_rules.py → _data_model/models/_base_verified.py} +13 -13
- cognite/neat/core/{_rules → _data_model}/models/_types.py +13 -13
- cognite/neat/core/_data_model/models/conceptual/__init__.py +25 -0
- cognite/neat/core/{_rules/models/information/_rules_input.py → _data_model/models/conceptual/_unverified.py} +46 -43
- cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py +93 -79
- cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py} +83 -83
- cognite/neat/core/{_rules → _data_model}/models/data_types.py +4 -4
- cognite/neat/core/{_rules → _data_model}/models/entities/__init__.py +8 -8
- cognite/neat/core/{_rules → _data_model}/models/entities/_loaders.py +12 -11
- cognite/neat/core/{_rules → _data_model}/models/entities/_multi_value.py +7 -7
- cognite/neat/core/{_rules → _data_model}/models/entities/_single_value.py +45 -39
- cognite/neat/core/{_rules → _data_model}/models/entities/_types.py +9 -3
- cognite/neat/core/{_rules → _data_model}/models/entities/_wrapped.py +3 -3
- cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.py +12 -9
- cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_exporter.py +83 -64
- cognite/neat/core/{_rules/models/dms/_rules_input.py → _data_model/models/physical/_unverified.py} +56 -44
- cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_validation.py +20 -17
- cognite/neat/core/{_rules/models/dms/_rules.py → _data_model/models/physical/_verified.py} +79 -71
- cognite/neat/core/{_rules → _data_model}/transformers/__init__.py +27 -23
- cognite/neat/core/{_rules → _data_model}/transformers/_base.py +29 -19
- cognite/neat/core/{_rules → _data_model}/transformers/_converters.py +758 -659
- cognite/neat/core/{_rules → _data_model}/transformers/_mapping.py +79 -60
- cognite/neat/core/_data_model/transformers/_verification.py +120 -0
- cognite/neat/core/{_graph → _instances}/extractors/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_base.py +1 -1
- cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_classic.py +17 -11
- cognite/neat/core/{_graph → _instances}/extractors/_dms_graph.py +47 -39
- cognite/neat/core/{_graph → _instances}/extractors/_mock_graph_generator.py +102 -99
- cognite/neat/core/{_graph → _instances}/extractors/_rdf_file.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_base.py +2 -2
- cognite/neat/core/{_graph → _instances}/loaders/_rdf2dms.py +16 -14
- cognite/neat/core/{_graph → _instances}/transformers/_base.py +7 -4
- cognite/neat/core/{_graph → _instances}/transformers/_classic_cdf.py +1 -1
- cognite/neat/core/{_graph → _instances}/transformers/_value_type.py +2 -6
- cognite/neat/core/_issues/_base.py +4 -4
- cognite/neat/core/_issues/errors/__init__.py +2 -2
- cognite/neat/core/_issues/errors/_wrapper.py +2 -2
- cognite/neat/core/_issues/warnings/__init__.py +2 -0
- cognite/neat/core/_issues/warnings/_models.py +4 -4
- cognite/neat/core/_issues/warnings/_properties.py +7 -0
- cognite/neat/core/_store/__init__.py +3 -3
- cognite/neat/core/_store/{_rules_store.py → _data_model.py} +128 -121
- cognite/neat/core/_store/{_graph_store.py → _instance.py} +7 -8
- cognite/neat/core/_store/_provenance.py +2 -2
- cognite/neat/core/_store/exceptions.py +4 -4
- cognite/neat/core/_utils/rdf_.py +14 -0
- cognite/neat/core/_utils/spreadsheet.py +1 -1
- cognite/neat/core/_utils/text.py +2 -2
- cognite/neat/session/_base.py +29 -25
- cognite/neat/session/_drop.py +3 -3
- cognite/neat/session/_fix.py +2 -2
- cognite/neat/session/_inspect.py +5 -5
- cognite/neat/session/_mapping.py +11 -9
- cognite/neat/session/_prepare.py +4 -4
- cognite/neat/session/_read.py +15 -15
- cognite/neat/session/_set.py +5 -5
- cognite/neat/session/_show.py +11 -11
- cognite/neat/session/_state.py +17 -17
- cognite/neat/session/_subset.py +14 -11
- cognite/neat/session/_template.py +19 -19
- cognite/neat/session/_to.py +21 -21
- cognite/neat/session/_wizard.py +1 -1
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
- cognite_neat-0.121.2.dist-info/RECORD +189 -0
- cognite/neat/core/_rules/_shared.py +0 -43
- cognite/neat/core/_rules/analysis/__init__.py +0 -3
- cognite/neat/core/_rules/exporters/_validation.py +0 -14
- cognite/neat/core/_rules/models/__init__.py +0 -34
- cognite/neat/core/_rules/models/dms/__init__.py +0 -32
- cognite/neat/core/_rules/models/information/__init__.py +0 -20
- cognite/neat/core/_rules/transformers/_verification.py +0 -111
- cognite_neat-0.121.0.dist-info/RECORD +0 -187
- /cognite/neat/core/{_graph → _data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/classic_model.xlsx +0 -0
- /cognite/neat/core/{_rules/catalog/info-rules-imf.xlsx → _data_model/catalog/conceptual-imf-data-model.xlsx} +0 -0
- /cognite/neat/core/{_rules → _data_model}/catalog/hello_world_pump.xlsx +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/__init__.py +0 -0
- /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/_unit_lookup.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/importers/_rdf/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/entities/_constants.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/__init__.py +0 -0
- /cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.yaml +0 -0
- /cognite/neat/core/{_graph/extractors/_classic_cdf → _instances}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_shared.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/_tracking/log.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44.xml +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/__init__.py +0 -0
- /cognite/neat/core/{_rules → _instances/extractors/_classic_cdf}/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_assets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_data_sets.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_events.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_files.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_labels.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_relationships.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_sequences.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_timeseries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dict.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_dms.py +0 -0
- /cognite/neat/core/{_graph → _instances}/extractors/_raw.py +0 -0
- /cognite/neat/core/{_graph → _instances}/loaders/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_base.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_queries.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_select.py +0 -0
- /cognite/neat/core/{_graph → _instances}/queries/_update.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/__init__.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_prune_graph.py +0 -0
- /cognite/neat/core/{_graph → _instances}/transformers/_rdfpath.py +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
- {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -29,33 +29,30 @@ from cognite.neat.core._constants import (
|
|
|
29
29
|
DMS_RESERVED_PROPERTIES,
|
|
30
30
|
get_default_prefixes_and_namespaces,
|
|
31
31
|
)
|
|
32
|
-
from cognite.neat.core.
|
|
33
|
-
from cognite.neat.core.
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
PropertyOverwritingWarning,
|
|
38
|
-
)
|
|
39
|
-
from cognite.neat.core._issues.warnings._models import (
|
|
40
|
-
SolutionModelBuildOnTopOfCDMWarning,
|
|
32
|
+
from cognite.neat.core._data_model._constants import PATTERNS, get_reserved_words
|
|
33
|
+
from cognite.neat.core._data_model._shared import (
|
|
34
|
+
ImportedDataModel,
|
|
35
|
+
ImportedUnverifiedDataModel,
|
|
36
|
+
VerifiedDataModel,
|
|
41
37
|
)
|
|
42
|
-
from cognite.neat.core.
|
|
43
|
-
from cognite.neat.core.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
)
|
|
48
|
-
from cognite.neat.core._rules.analysis import RulesAnalysis
|
|
49
|
-
from cognite.neat.core._rules.importers import DMSImporter
|
|
50
|
-
from cognite.neat.core._rules.models import (
|
|
51
|
-
DMSInputRules,
|
|
52
|
-
DMSRules,
|
|
53
|
-
InformationInputRules,
|
|
54
|
-
InformationRules,
|
|
38
|
+
from cognite.neat.core._data_model.analysis import DataModelAnalysis
|
|
39
|
+
from cognite.neat.core._data_model.importers import DMSImporter
|
|
40
|
+
from cognite.neat.core._data_model.models import (
|
|
41
|
+
ConceptualDataModel,
|
|
42
|
+
PhysicalDataModel,
|
|
55
43
|
SheetList,
|
|
44
|
+
UnverifiedConceptualDataModel,
|
|
45
|
+
UnverifiedPhysicalDataModel,
|
|
56
46
|
data_types,
|
|
57
47
|
)
|
|
58
|
-
from cognite.neat.core.
|
|
48
|
+
from cognite.neat.core._data_model.models.conceptual import (
|
|
49
|
+
Concept,
|
|
50
|
+
ConceptualMetadata,
|
|
51
|
+
ConceptualProperty,
|
|
52
|
+
UnverifiedConcept,
|
|
53
|
+
UnverifiedConceptualProperty,
|
|
54
|
+
)
|
|
55
|
+
from cognite.neat.core._data_model.models.data_types import (
|
|
59
56
|
AnyURI,
|
|
60
57
|
DataType,
|
|
61
58
|
Enum,
|
|
@@ -63,30 +60,37 @@ from cognite.neat.core._rules.models.data_types import (
|
|
|
63
60
|
String,
|
|
64
61
|
Timeseries,
|
|
65
62
|
)
|
|
66
|
-
from cognite.neat.core.
|
|
67
|
-
|
|
68
|
-
DMSProperty,
|
|
69
|
-
DMSValidation,
|
|
70
|
-
DMSView,
|
|
71
|
-
)
|
|
72
|
-
from cognite.neat.core._rules.models.dms._rules import DMSContainer, DMSEnum, DMSNode
|
|
73
|
-
from cognite.neat.core._rules.models.entities import (
|
|
74
|
-
ClassEntity,
|
|
63
|
+
from cognite.neat.core._data_model.models.entities import (
|
|
64
|
+
ConceptEntity,
|
|
75
65
|
ContainerEntity,
|
|
76
|
-
DMSUnknownEntity,
|
|
77
66
|
EdgeEntity,
|
|
78
67
|
HasDataFilter,
|
|
79
68
|
MultiValueTypeInfo,
|
|
69
|
+
PhysicalUnknownEntity,
|
|
80
70
|
ReverseConnectionEntity,
|
|
81
71
|
UnknownEntity,
|
|
82
72
|
ViewEntity,
|
|
83
73
|
)
|
|
84
|
-
from cognite.neat.core.
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
74
|
+
from cognite.neat.core._data_model.models.physical import (
|
|
75
|
+
PhysicalMetadata,
|
|
76
|
+
PhysicalProperty,
|
|
77
|
+
PhysicalValidation,
|
|
78
|
+
PhysicalView,
|
|
79
|
+
)
|
|
80
|
+
from cognite.neat.core._data_model.models.physical._verified import (
|
|
81
|
+
PhysicalContainer,
|
|
82
|
+
PhysicalEnum,
|
|
83
|
+
PhysicalNodeType,
|
|
84
|
+
)
|
|
85
|
+
from cognite.neat.core._issues import IssueList
|
|
86
|
+
from cognite.neat.core._issues._factory import from_pydantic_errors
|
|
87
|
+
from cognite.neat.core._issues.errors import CDFMissingClientError, NeatValueError
|
|
88
|
+
from cognite.neat.core._issues.warnings import (
|
|
89
|
+
NeatValueWarning,
|
|
90
|
+
PropertyOverwritingWarning,
|
|
91
|
+
)
|
|
92
|
+
from cognite.neat.core._issues.warnings._models import (
|
|
93
|
+
SolutionModelBuildOnTopOfCDMWarning,
|
|
90
94
|
)
|
|
91
95
|
from cognite.neat.core._utils.rdf_ import get_inheritance_path
|
|
92
96
|
from cognite.neat.core._utils.spreadsheet import SpreadsheetRead
|
|
@@ -98,24 +102,32 @@ from cognite.neat.core._utils.text import (
|
|
|
98
102
|
to_words,
|
|
99
103
|
)
|
|
100
104
|
|
|
101
|
-
from ._base import
|
|
102
|
-
|
|
105
|
+
from ._base import (
|
|
106
|
+
DataModelTransformer,
|
|
107
|
+
T_VerifiedIn,
|
|
108
|
+
T_VerifiedOut,
|
|
109
|
+
VerifiedDataModelTransformer,
|
|
110
|
+
)
|
|
111
|
+
from ._verification import VerifyPhysicalDataModel
|
|
103
112
|
|
|
104
|
-
T_InputInRules = TypeVar("T_InputInRules", bound=
|
|
105
|
-
T_InputOutRules = TypeVar("T_InputOutRules", bound=
|
|
113
|
+
T_InputInRules = TypeVar("T_InputInRules", bound=ImportedUnverifiedDataModel)
|
|
114
|
+
T_InputOutRules = TypeVar("T_InputOutRules", bound=ImportedUnverifiedDataModel)
|
|
106
115
|
|
|
107
116
|
|
|
108
|
-
class ConversionTransformer(
|
|
117
|
+
class ConversionTransformer(VerifiedDataModelTransformer[T_VerifiedIn, T_VerifiedOut], ABC):
|
|
109
118
|
"""Base class for all conversion transformers."""
|
|
110
119
|
|
|
111
120
|
...
|
|
112
121
|
|
|
113
122
|
|
|
114
|
-
class ToDMSCompliantEntities(
|
|
115
|
-
|
|
123
|
+
class ToDMSCompliantEntities(
|
|
124
|
+
DataModelTransformer[
|
|
125
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
126
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
127
|
+
]
|
|
128
|
+
):
|
|
129
|
+
"""Makes concept and property ids compliant with DMS regex restrictions.
|
|
116
130
|
|
|
117
|
-
This is typically used with importers from arbitrary sources to ensure that classes and properties have valid
|
|
118
|
-
names.
|
|
119
131
|
|
|
120
132
|
Args:
|
|
121
133
|
rename_warning: How to handle renaming of entities that are not compliant with the Information Model.
|
|
@@ -130,70 +142,74 @@ class ToDMSCompliantEntities(RulesTransformer[ReadRules[InformationInputRules],
|
|
|
130
142
|
def description(self) -> str:
|
|
131
143
|
return "Ensures that all entities are compliant with the Information Model."
|
|
132
144
|
|
|
133
|
-
def transform(
|
|
134
|
-
|
|
135
|
-
|
|
145
|
+
def transform(
|
|
146
|
+
self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]
|
|
147
|
+
) -> ImportedDataModel[UnverifiedConceptualDataModel]:
|
|
148
|
+
if data_model.unverified_data_model is None:
|
|
149
|
+
return data_model
|
|
136
150
|
# Doing dump to obtain a copy, and ensure that all entities are created. Input allows
|
|
137
151
|
# string for entities, the dump call will convert these to entities.
|
|
138
|
-
dumped =
|
|
139
|
-
copy =
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
for
|
|
143
|
-
|
|
144
|
-
if not PATTERNS.view_id_compliance.match(
|
|
145
|
-
new_suffix = self.
|
|
152
|
+
dumped = data_model.unverified_data_model.dump()
|
|
153
|
+
copy = UnverifiedConceptualDataModel.load(dumped)
|
|
154
|
+
|
|
155
|
+
new_by_old_concept_suffix: dict[str, str] = {}
|
|
156
|
+
for concept in copy.concepts:
|
|
157
|
+
concept_entity = cast(ConceptEntity, concept.concept) # Safe due to the dump above
|
|
158
|
+
if not PATTERNS.view_id_compliance.match(concept_entity.suffix):
|
|
159
|
+
new_suffix = self._fix_concept_suffix(concept_entity.suffix)
|
|
146
160
|
if self._renaming == "raise":
|
|
147
161
|
warnings.warn(
|
|
148
|
-
NeatValueWarning(f"Invalid class name {
|
|
162
|
+
NeatValueWarning(f"Invalid class name {concept_entity.suffix!r}.Renaming to {new_suffix}"),
|
|
149
163
|
stacklevel=2,
|
|
150
164
|
)
|
|
151
|
-
|
|
165
|
+
concept.concept.suffix = new_suffix # type: ignore[union-attr]
|
|
152
166
|
|
|
153
|
-
for
|
|
154
|
-
if
|
|
155
|
-
for i, parent in enumerate(
|
|
156
|
-
if isinstance(parent,
|
|
157
|
-
|
|
167
|
+
for concept in copy.concepts:
|
|
168
|
+
if concept.implements:
|
|
169
|
+
for i, parent in enumerate(concept.implements):
|
|
170
|
+
if isinstance(parent, ConceptEntity) and parent.suffix in new_by_old_concept_suffix:
|
|
171
|
+
concept.implements[i].suffix = new_by_old_concept_suffix[parent.suffix] # type: ignore[union-attr]
|
|
158
172
|
|
|
159
173
|
for prop in copy.properties:
|
|
160
|
-
if not PATTERNS.
|
|
174
|
+
if not PATTERNS.physical_property_id_compliance.match(prop.property_):
|
|
161
175
|
new_property = self._fix_property(prop.property_)
|
|
162
176
|
if self._renaming == "warning":
|
|
163
177
|
warnings.warn(
|
|
164
178
|
NeatValueWarning(
|
|
165
|
-
f"Invalid property name {prop.
|
|
179
|
+
f"Invalid property name {prop.concept.suffix}.{prop.property_!r}."
|
|
180
|
+
f" Renaming to {new_property}"
|
|
181
|
+
# type: ignore[union-attr]
|
|
166
182
|
),
|
|
167
183
|
stacklevel=2,
|
|
168
184
|
)
|
|
169
185
|
prop.property_ = new_property
|
|
170
186
|
|
|
171
|
-
if isinstance(prop.
|
|
172
|
-
prop.
|
|
187
|
+
if isinstance(prop.concept, ConceptEntity) and prop.concept.suffix in new_by_old_concept_suffix:
|
|
188
|
+
prop.concept.suffix = new_by_old_concept_suffix[prop.concept.suffix]
|
|
173
189
|
|
|
174
|
-
if isinstance(prop.value_type,
|
|
175
|
-
prop.value_type.suffix =
|
|
190
|
+
if isinstance(prop.value_type, ConceptEntity) and prop.value_type.suffix in new_by_old_concept_suffix:
|
|
191
|
+
prop.value_type.suffix = new_by_old_concept_suffix[prop.value_type.suffix]
|
|
176
192
|
|
|
177
193
|
if isinstance(prop.value_type, MultiValueTypeInfo):
|
|
178
194
|
for i, value_type in enumerate(prop.value_type.types):
|
|
179
|
-
if isinstance(value_type,
|
|
180
|
-
prop.value_type.types[i].suffix =
|
|
195
|
+
if isinstance(value_type, ConceptEntity) and value_type.suffix in new_by_old_concept_suffix:
|
|
196
|
+
prop.value_type.types[i].suffix = new_by_old_concept_suffix[value_type.suffix] # type: ignore[union-attr]
|
|
181
197
|
|
|
182
|
-
return
|
|
198
|
+
return ImportedDataModel(unverified_data_model=copy, context=data_model.context)
|
|
183
199
|
|
|
184
200
|
@cached_property
|
|
185
|
-
def
|
|
186
|
-
return set(get_reserved_words("
|
|
201
|
+
def _reserved_concept_words(self) -> set[str]:
|
|
202
|
+
return set(get_reserved_words("concept"))
|
|
187
203
|
|
|
188
204
|
@cached_property
|
|
189
205
|
def _reserved_property_words(self) -> set[str]:
|
|
190
206
|
return set(get_reserved_words("property"))
|
|
191
207
|
|
|
192
|
-
def
|
|
193
|
-
if suffix in self.
|
|
208
|
+
def _fix_concept_suffix(self, suffix: str) -> str:
|
|
209
|
+
if suffix in self._reserved_concept_words:
|
|
194
210
|
return f"My{suffix}"
|
|
195
211
|
suffix = urllib.parse.unquote(suffix)
|
|
196
|
-
suffix = NamingStandardization.
|
|
212
|
+
suffix = NamingStandardization.standardize_concept_str(suffix)
|
|
197
213
|
if len(suffix) > 252:
|
|
198
214
|
suffix = suffix[:252]
|
|
199
215
|
return suffix
|
|
@@ -208,7 +224,7 @@ class ToDMSCompliantEntities(RulesTransformer[ReadRules[InformationInputRules],
|
|
|
208
224
|
return property_
|
|
209
225
|
|
|
210
226
|
|
|
211
|
-
class StandardizeSpaceAndVersion(
|
|
227
|
+
class StandardizeSpaceAndVersion(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]): # type: ignore[misc]
|
|
212
228
|
"""This transformer standardizes the space and version of the DMSRules.
|
|
213
229
|
|
|
214
230
|
typically used to ensure all the views are moved to the same version as the data model.
|
|
@@ -219,8 +235,8 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
219
235
|
def description(self) -> str:
|
|
220
236
|
return "Ensures uniform version and space of the views belonging to the data model."
|
|
221
237
|
|
|
222
|
-
def transform(self,
|
|
223
|
-
copy =
|
|
238
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
239
|
+
copy = data_model.model_copy(deep=True)
|
|
224
240
|
|
|
225
241
|
space = copy.metadata.space
|
|
226
242
|
version = copy.metadata.version
|
|
@@ -229,7 +245,7 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
229
245
|
copy.properties = self._standardize_properties(copy.properties, space, version)
|
|
230
246
|
return copy
|
|
231
247
|
|
|
232
|
-
def _standardize_views(self, views: SheetList[
|
|
248
|
+
def _standardize_views(self, views: SheetList[PhysicalView], space: str, version: str) -> SheetList[PhysicalView]:
|
|
233
249
|
for view in views:
|
|
234
250
|
if view.view.space not in COGNITE_SPACES:
|
|
235
251
|
view.view.version = version
|
|
@@ -243,8 +259,8 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
243
259
|
return views
|
|
244
260
|
|
|
245
261
|
def _standardize_properties(
|
|
246
|
-
self, properties: SheetList[
|
|
247
|
-
) -> SheetList[
|
|
262
|
+
self, properties: SheetList[PhysicalProperty], space: str, version: str
|
|
263
|
+
) -> SheetList[PhysicalProperty]:
|
|
248
264
|
for property_ in properties:
|
|
249
265
|
if property_.view.space not in COGNITE_SPACES:
|
|
250
266
|
property_.view.version = version
|
|
@@ -267,17 +283,17 @@ class StandardizeSpaceAndVersion(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
267
283
|
return properties
|
|
268
284
|
|
|
269
285
|
|
|
270
|
-
class ToCompliantEntities(
|
|
271
|
-
"""Converts input
|
|
286
|
+
class ToCompliantEntities(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]): # type: ignore[misc]
|
|
287
|
+
"""Converts input data_model to data_model with compliant entity IDs that match regex patters used
|
|
272
288
|
by DMS schema components."""
|
|
273
289
|
|
|
274
290
|
@property
|
|
275
291
|
def description(self) -> str:
|
|
276
292
|
return "Ensures externalIDs are compliant with CDF"
|
|
277
293
|
|
|
278
|
-
def transform(self,
|
|
279
|
-
copy =
|
|
280
|
-
copy.
|
|
294
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
295
|
+
copy = data_model.model_copy(deep=True)
|
|
296
|
+
copy.concepts = self._fix_concepts(copy.concepts)
|
|
281
297
|
copy.properties = self._fix_properties(copy.properties)
|
|
282
298
|
return copy
|
|
283
299
|
|
|
@@ -296,30 +312,30 @@ class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, Information
|
|
|
296
312
|
return re.sub(r"[^a-zA-Z0-9]+", "_", entity)
|
|
297
313
|
|
|
298
314
|
@classmethod
|
|
299
|
-
def
|
|
300
|
-
if isinstance(
|
|
301
|
-
|
|
302
|
-
prefix=cls._fix_entity(
|
|
303
|
-
suffix=cls._fix_entity(
|
|
315
|
+
def _fix_concept(cls, concept: ConceptEntity) -> ConceptEntity:
|
|
316
|
+
if isinstance(concept, ConceptEntity) and type(concept.prefix) is str:
|
|
317
|
+
concept = ConceptEntity(
|
|
318
|
+
prefix=cls._fix_entity(concept.prefix),
|
|
319
|
+
suffix=cls._fix_entity(concept.suffix),
|
|
304
320
|
)
|
|
305
321
|
|
|
306
|
-
return
|
|
322
|
+
return concept
|
|
307
323
|
|
|
308
324
|
@classmethod
|
|
309
325
|
def _fix_value_type(
|
|
310
|
-
cls, value_type: DataType |
|
|
311
|
-
) -> DataType |
|
|
312
|
-
fixed_value_type: DataType |
|
|
326
|
+
cls, value_type: DataType | ConceptEntity | MultiValueTypeInfo
|
|
327
|
+
) -> DataType | ConceptEntity | MultiValueTypeInfo:
|
|
328
|
+
fixed_value_type: DataType | ConceptEntity | MultiValueTypeInfo
|
|
313
329
|
|
|
314
330
|
# value type specified as MultiValueTypeInfo
|
|
315
331
|
if isinstance(value_type, MultiValueTypeInfo):
|
|
316
332
|
fixed_value_type = MultiValueTypeInfo(
|
|
317
|
-
types=[cast(DataType |
|
|
333
|
+
types=[cast(DataType | ConceptEntity, cls._fix_value_type(type_)) for type_ in value_type.types],
|
|
318
334
|
)
|
|
319
335
|
|
|
320
336
|
# value type specified as ClassEntity instance
|
|
321
|
-
elif isinstance(value_type,
|
|
322
|
-
fixed_value_type = cls.
|
|
337
|
+
elif isinstance(value_type, ConceptEntity):
|
|
338
|
+
fixed_value_type = cls._fix_concept(value_type)
|
|
323
339
|
|
|
324
340
|
# this is a DataType instance but also we should default to original value
|
|
325
341
|
else:
|
|
@@ -328,18 +344,18 @@ class ToCompliantEntities(VerifiedRulesTransformer[InformationRules, Information
|
|
|
328
344
|
return fixed_value_type
|
|
329
345
|
|
|
330
346
|
@classmethod
|
|
331
|
-
def
|
|
332
|
-
fixed_definitions = SheetList[
|
|
347
|
+
def _fix_concepts(cls, definitions: SheetList[Concept]) -> SheetList[Concept]:
|
|
348
|
+
fixed_definitions = SheetList[Concept]()
|
|
333
349
|
for definition in definitions:
|
|
334
|
-
definition.
|
|
350
|
+
definition.concept = cls._fix_concept(definition.concept)
|
|
335
351
|
fixed_definitions.append(definition)
|
|
336
352
|
return fixed_definitions
|
|
337
353
|
|
|
338
354
|
@classmethod
|
|
339
|
-
def _fix_properties(cls, definitions: SheetList[
|
|
340
|
-
fixed_definitions = SheetList[
|
|
355
|
+
def _fix_properties(cls, definitions: SheetList[ConceptualProperty]) -> SheetList[ConceptualProperty]:
|
|
356
|
+
fixed_definitions = SheetList[ConceptualProperty]()
|
|
341
357
|
for definition in definitions:
|
|
342
|
-
definition.
|
|
358
|
+
definition.concept = cls._fix_concept(definition.concept)
|
|
343
359
|
definition.property_ = cls._fix_entity(definition.property_)
|
|
344
360
|
definition.value_type = cls._fix_value_type(definition.value_type)
|
|
345
361
|
fixed_definitions.append(definition)
|
|
@@ -357,20 +373,20 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
357
373
|
return f"Prefixes all entities with {self._prefix!r} prefix if they are in the same space as data model."
|
|
358
374
|
|
|
359
375
|
@overload
|
|
360
|
-
def transform(self,
|
|
376
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel: ...
|
|
361
377
|
|
|
362
378
|
@overload
|
|
363
|
-
def transform(self,
|
|
379
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel: ...
|
|
364
380
|
|
|
365
|
-
def transform(self,
|
|
366
|
-
copy:
|
|
381
|
+
def transform(self, data_model: ConceptualDataModel | PhysicalDataModel) -> ConceptualDataModel | PhysicalDataModel:
|
|
382
|
+
copy: ConceptualDataModel | PhysicalDataModel = data_model.model_copy(deep=True)
|
|
367
383
|
|
|
368
384
|
# Case: Prefix Information Rules
|
|
369
|
-
if isinstance(copy,
|
|
385
|
+
if isinstance(copy, ConceptualDataModel):
|
|
370
386
|
# prefix classes
|
|
371
|
-
for cls in copy.
|
|
372
|
-
if cls.
|
|
373
|
-
cls.
|
|
387
|
+
for cls in copy.concepts:
|
|
388
|
+
if cls.concept.prefix == copy.metadata.prefix:
|
|
389
|
+
cls.concept = self._with_prefix(cls.concept)
|
|
374
390
|
|
|
375
391
|
if cls.implements:
|
|
376
392
|
# prefix parents
|
|
@@ -379,21 +395,21 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
379
395
|
cls.implements[i] = self._with_prefix(parent_class)
|
|
380
396
|
|
|
381
397
|
for prop in copy.properties:
|
|
382
|
-
if prop.
|
|
383
|
-
prop.
|
|
398
|
+
if prop.concept.prefix == copy.metadata.prefix:
|
|
399
|
+
prop.concept = self._with_prefix(prop.concept)
|
|
384
400
|
|
|
385
401
|
# value type property is not multi and it is ClassEntity
|
|
386
402
|
|
|
387
|
-
if isinstance(prop.value_type,
|
|
388
|
-
prop.value_type = self._with_prefix(cast(
|
|
403
|
+
if isinstance(prop.value_type, ConceptEntity) and prop.value_type.prefix == copy.metadata.prefix:
|
|
404
|
+
prop.value_type = self._with_prefix(cast(ConceptEntity, prop.value_type))
|
|
389
405
|
elif isinstance(prop.value_type, MultiValueTypeInfo):
|
|
390
406
|
for i, value_type in enumerate(prop.value_type.types):
|
|
391
|
-
if isinstance(value_type,
|
|
392
|
-
prop.value_type.types[i] = self._with_prefix(cast(
|
|
407
|
+
if isinstance(value_type, ConceptEntity) and value_type.prefix == copy.metadata.prefix:
|
|
408
|
+
prop.value_type.types[i] = self._with_prefix(cast(ConceptEntity, value_type))
|
|
393
409
|
return copy
|
|
394
410
|
|
|
395
411
|
# Case: Prefix DMS Rules
|
|
396
|
-
elif isinstance(copy,
|
|
412
|
+
elif isinstance(copy, PhysicalDataModel):
|
|
397
413
|
for view in copy.views:
|
|
398
414
|
if view.view.space == copy.metadata.space:
|
|
399
415
|
view.view = self._with_prefix(view.view)
|
|
@@ -403,15 +419,21 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
403
419
|
if parent_view.space == copy.metadata.space:
|
|
404
420
|
view.implements[i] = self._with_prefix(parent_view)
|
|
405
421
|
|
|
406
|
-
for
|
|
407
|
-
if
|
|
408
|
-
|
|
422
|
+
for physical_prop in copy.properties:
|
|
423
|
+
if physical_prop.view.space == copy.metadata.space:
|
|
424
|
+
physical_prop.view = self._with_prefix(physical_prop.view)
|
|
409
425
|
|
|
410
|
-
if
|
|
411
|
-
|
|
426
|
+
if (
|
|
427
|
+
isinstance(physical_prop.value_type, ViewEntity)
|
|
428
|
+
and physical_prop.value_type.space == copy.metadata.space
|
|
429
|
+
):
|
|
430
|
+
physical_prop.value_type = self._with_prefix(physical_prop.value_type)
|
|
412
431
|
|
|
413
|
-
if
|
|
414
|
-
|
|
432
|
+
if (
|
|
433
|
+
isinstance(physical_prop.container, ContainerEntity)
|
|
434
|
+
and physical_prop.container.space == copy.metadata.space
|
|
435
|
+
):
|
|
436
|
+
physical_prop.container = self._with_prefix(physical_prop.container)
|
|
415
437
|
|
|
416
438
|
if copy.containers:
|
|
417
439
|
for container in copy.containers:
|
|
@@ -419,10 +441,10 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
419
441
|
container.container = self._with_prefix(container.container)
|
|
420
442
|
return copy
|
|
421
443
|
|
|
422
|
-
raise NeatValueError(f"Unsupported
|
|
444
|
+
raise NeatValueError(f"Unsupported data_model type: {type(copy)}")
|
|
423
445
|
|
|
424
446
|
@overload
|
|
425
|
-
def _with_prefix(self, entity:
|
|
447
|
+
def _with_prefix(self, entity: ConceptEntity) -> ConceptEntity: ...
|
|
426
448
|
|
|
427
449
|
@overload
|
|
428
450
|
def _with_prefix(self, entity: ViewEntity) -> ViewEntity: ...
|
|
@@ -431,9 +453,9 @@ class PrefixEntities(ConversionTransformer): # type: ignore[type-var]
|
|
|
431
453
|
def _with_prefix(self, entity: ContainerEntity) -> ContainerEntity: ...
|
|
432
454
|
|
|
433
455
|
def _with_prefix(
|
|
434
|
-
self, entity: ViewEntity | ContainerEntity |
|
|
435
|
-
) -> ViewEntity | ContainerEntity |
|
|
436
|
-
if isinstance(entity, ViewEntity | ContainerEntity |
|
|
456
|
+
self, entity: ViewEntity | ContainerEntity | ConceptEntity
|
|
457
|
+
) -> ViewEntity | ContainerEntity | ConceptEntity:
|
|
458
|
+
if isinstance(entity, ViewEntity | ContainerEntity | ConceptEntity):
|
|
437
459
|
entity.suffix = f"{self._prefix}{entity.suffix}"
|
|
438
460
|
|
|
439
461
|
else:
|
|
@@ -450,61 +472,61 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
450
472
|
return "Sets views/classes/containers names to PascalCase and properties to camelCase."
|
|
451
473
|
|
|
452
474
|
@overload
|
|
453
|
-
def transform(self,
|
|
475
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel: ...
|
|
454
476
|
|
|
455
477
|
@overload
|
|
456
|
-
def transform(self,
|
|
457
|
-
|
|
458
|
-
def transform(self,
|
|
459
|
-
output =
|
|
460
|
-
if isinstance(output,
|
|
461
|
-
return self.
|
|
462
|
-
elif isinstance(output,
|
|
463
|
-
return self.
|
|
464
|
-
raise NeatValueError(f"Unsupported
|
|
465
|
-
|
|
466
|
-
def
|
|
467
|
-
|
|
468
|
-
for cls in
|
|
469
|
-
new_suffix = NamingStandardization.
|
|
470
|
-
|
|
471
|
-
cls.
|
|
472
|
-
|
|
473
|
-
for cls in
|
|
478
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel: ...
|
|
479
|
+
|
|
480
|
+
def transform(self, data_model: ConceptualDataModel | PhysicalDataModel) -> ConceptualDataModel | PhysicalDataModel:
|
|
481
|
+
output = data_model.model_copy(deep=True)
|
|
482
|
+
if isinstance(output, ConceptualDataModel):
|
|
483
|
+
return self._standardize_conceptual_data_model(output)
|
|
484
|
+
elif isinstance(output, PhysicalDataModel):
|
|
485
|
+
return self._standardize_physical_data_model(output)
|
|
486
|
+
raise NeatValueError(f"Unsupported data_model type: {type(output)}")
|
|
487
|
+
|
|
488
|
+
def _standardize_conceptual_data_model(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
489
|
+
new_by_old_concept_suffix: dict[str, str] = {}
|
|
490
|
+
for cls in data_model.concepts:
|
|
491
|
+
new_suffix = NamingStandardization.standardize_concept_str(cls.concept.suffix)
|
|
492
|
+
new_by_old_concept_suffix[cls.concept.suffix] = new_suffix
|
|
493
|
+
cls.concept.suffix = new_suffix
|
|
494
|
+
|
|
495
|
+
for cls in data_model.concepts:
|
|
474
496
|
if cls.implements:
|
|
475
497
|
for i, parent in enumerate(cls.implements):
|
|
476
|
-
if parent.suffix in
|
|
477
|
-
cls.implements[i].suffix =
|
|
498
|
+
if parent.suffix in new_by_old_concept_suffix:
|
|
499
|
+
cls.implements[i].suffix = new_by_old_concept_suffix[parent.suffix]
|
|
478
500
|
|
|
479
|
-
for prop in
|
|
501
|
+
for prop in data_model.properties:
|
|
480
502
|
prop.property_ = NamingStandardization.standardize_property_str(prop.property_)
|
|
481
|
-
if prop.
|
|
482
|
-
prop.
|
|
503
|
+
if prop.concept.suffix in new_by_old_concept_suffix:
|
|
504
|
+
prop.concept.suffix = new_by_old_concept_suffix[prop.concept.suffix]
|
|
483
505
|
|
|
484
|
-
if isinstance(prop.value_type,
|
|
485
|
-
prop.value_type.suffix =
|
|
506
|
+
if isinstance(prop.value_type, ConceptEntity) and prop.value_type.suffix in new_by_old_concept_suffix:
|
|
507
|
+
prop.value_type.suffix = new_by_old_concept_suffix[prop.value_type.suffix]
|
|
486
508
|
|
|
487
509
|
if isinstance(prop.value_type, MultiValueTypeInfo):
|
|
488
510
|
for i, value_type in enumerate(prop.value_type.types):
|
|
489
|
-
if isinstance(value_type,
|
|
490
|
-
prop.value_type.types[i].suffix =
|
|
511
|
+
if isinstance(value_type, ConceptEntity) and value_type.suffix in new_by_old_concept_suffix:
|
|
512
|
+
prop.value_type.types[i].suffix = new_by_old_concept_suffix[value_type.suffix] # type: ignore[union-attr]
|
|
491
513
|
|
|
492
|
-
return
|
|
514
|
+
return data_model
|
|
493
515
|
|
|
494
|
-
def
|
|
516
|
+
def _standardize_physical_data_model(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
495
517
|
new_by_old_view: dict[str, str] = {}
|
|
496
|
-
for view in
|
|
497
|
-
new_suffix = NamingStandardization.
|
|
518
|
+
for view in data_model.views:
|
|
519
|
+
new_suffix = NamingStandardization.standardize_concept_str(view.view.suffix)
|
|
498
520
|
new_by_old_view[view.view.suffix] = new_suffix
|
|
499
521
|
view.view.suffix = new_suffix
|
|
500
522
|
new_by_old_container: dict[str, str] = {}
|
|
501
|
-
if
|
|
502
|
-
for container in
|
|
503
|
-
new_suffix = NamingStandardization.
|
|
523
|
+
if data_model.containers:
|
|
524
|
+
for container in data_model.containers:
|
|
525
|
+
new_suffix = NamingStandardization.standardize_concept_str(container.container.suffix)
|
|
504
526
|
new_by_old_container[container.container.suffix] = new_suffix
|
|
505
527
|
container.container.suffix = new_suffix
|
|
506
528
|
|
|
507
|
-
for view in
|
|
529
|
+
for view in data_model.views:
|
|
508
530
|
if view.implements:
|
|
509
531
|
for i, parent in enumerate(view.implements):
|
|
510
532
|
if parent.suffix in new_by_old_view:
|
|
@@ -515,14 +537,14 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
515
537
|
view.filter_.inner[i].suffix = new_by_old_container[item.suffix]
|
|
516
538
|
if isinstance(item, ViewEntity) and item.suffix in new_by_old_view:
|
|
517
539
|
view.filter_.inner[i].suffix = new_by_old_view[item.suffix]
|
|
518
|
-
if
|
|
519
|
-
for container in
|
|
540
|
+
if data_model.containers:
|
|
541
|
+
for container in data_model.containers:
|
|
520
542
|
if container.constraint:
|
|
521
543
|
for i, constraint in enumerate(container.constraint):
|
|
522
544
|
if constraint.suffix in new_by_old_container:
|
|
523
545
|
container.constraint[i].suffix = new_by_old_container[constraint.suffix]
|
|
524
546
|
new_property_by_view_by_old_property: dict[ViewEntity, dict[str, str]] = defaultdict(dict)
|
|
525
|
-
for prop in
|
|
547
|
+
for prop in data_model.properties:
|
|
526
548
|
if prop.view.suffix in new_by_old_view:
|
|
527
549
|
prop.view.suffix = new_by_old_view[prop.view.suffix]
|
|
528
550
|
new_view_property = NamingStandardization.standardize_property_str(prop.view_property)
|
|
@@ -540,7 +562,7 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
540
562
|
prop.container.suffix = new_by_old_container[prop.container.suffix]
|
|
541
563
|
if prop.container_property:
|
|
542
564
|
prop.container_property = NamingStandardization.standardize_property_str(prop.container_property)
|
|
543
|
-
for prop in
|
|
565
|
+
for prop in data_model.properties:
|
|
544
566
|
if (
|
|
545
567
|
isinstance(prop.connection, ReverseConnectionEntity)
|
|
546
568
|
and isinstance(prop.value_type, ViewEntity)
|
|
@@ -549,11 +571,11 @@ class StandardizeNaming(ConversionTransformer):
|
|
|
549
571
|
new_by_old_property = new_property_by_view_by_old_property[prop.value_type]
|
|
550
572
|
if prop.connection.property_ in new_by_old_property:
|
|
551
573
|
prop.connection.property_ = new_by_old_property[prop.connection.property_]
|
|
552
|
-
return
|
|
574
|
+
return data_model
|
|
553
575
|
|
|
554
576
|
|
|
555
|
-
class
|
|
556
|
-
"""Converts
|
|
577
|
+
class ConceptualToPhysical(ConversionTransformer[ConceptualDataModel, PhysicalDataModel]):
|
|
578
|
+
"""Converts conceptual to physical data model."""
|
|
557
579
|
|
|
558
580
|
def __init__(
|
|
559
581
|
self,
|
|
@@ -565,42 +587,42 @@ class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
|
565
587
|
self.reserved_properties = reserved_properties
|
|
566
588
|
self.client = client
|
|
567
589
|
|
|
568
|
-
def transform(self,
|
|
569
|
-
return
|
|
590
|
+
def transform(self, data_model: ConceptualDataModel) -> PhysicalDataModel:
|
|
591
|
+
return _ConceptualDataModelConverter(data_model, self.client).as_physical_data_model(
|
|
570
592
|
self.ignore_undefined_value_types, self.reserved_properties
|
|
571
593
|
)
|
|
572
594
|
|
|
573
595
|
|
|
574
|
-
class
|
|
575
|
-
"""Converts
|
|
596
|
+
class PhysicalToConceptual(ConversionTransformer[PhysicalDataModel, ConceptualDataModel]):
|
|
597
|
+
"""Converts Physical to Conceptual data model."""
|
|
576
598
|
|
|
577
599
|
def __init__(self, instance_namespace: Namespace | None = None):
|
|
578
600
|
self.instance_namespace = instance_namespace
|
|
579
601
|
|
|
580
|
-
def transform(self,
|
|
581
|
-
return _DMSRulesConverter(
|
|
602
|
+
def transform(self, data_model: PhysicalDataModel) -> ConceptualDataModel:
|
|
603
|
+
return _DMSRulesConverter(data_model, self.instance_namespace).as_conceptual_data_model()
|
|
582
604
|
|
|
583
605
|
|
|
584
|
-
class ConvertToRules(ConversionTransformer[
|
|
585
|
-
"""Converts any
|
|
606
|
+
class ConvertToRules(ConversionTransformer[VerifiedDataModel, VerifiedDataModel]):
|
|
607
|
+
"""Converts any data_model to any data_model."""
|
|
586
608
|
|
|
587
|
-
def __init__(self, out_cls: type[
|
|
609
|
+
def __init__(self, out_cls: type[VerifiedDataModel]):
|
|
588
610
|
self._out_cls = out_cls
|
|
589
611
|
|
|
590
|
-
def transform(self,
|
|
591
|
-
if isinstance(
|
|
592
|
-
return
|
|
593
|
-
if isinstance(
|
|
594
|
-
return
|
|
595
|
-
if isinstance(
|
|
596
|
-
return
|
|
597
|
-
raise ValueError(f"Unsupported conversion from {type(
|
|
612
|
+
def transform(self, data_model: VerifiedDataModel) -> VerifiedDataModel:
|
|
613
|
+
if isinstance(data_model, self._out_cls):
|
|
614
|
+
return data_model
|
|
615
|
+
if isinstance(data_model, ConceptualDataModel) and self._out_cls is PhysicalDataModel:
|
|
616
|
+
return ConceptualToPhysical().transform(data_model)
|
|
617
|
+
if isinstance(data_model, PhysicalDataModel) and self._out_cls is ConceptualDataModel:
|
|
618
|
+
return PhysicalToConceptual().transform(data_model)
|
|
619
|
+
raise ValueError(f"Unsupported conversion from {type(data_model)} to {self._out_cls}")
|
|
598
620
|
|
|
599
621
|
|
|
600
|
-
_T_Entity = TypeVar("_T_Entity", bound=
|
|
622
|
+
_T_Entity = TypeVar("_T_Entity", bound=ConceptEntity | ViewEntity)
|
|
601
623
|
|
|
602
624
|
|
|
603
|
-
class SetIDDMSModel(
|
|
625
|
+
class SetIDDMSModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
604
626
|
def __init__(self, new_id: DataModelId | tuple[str, str, str], name: str | None = None):
|
|
605
627
|
self.new_id = DataModelId.load(new_id)
|
|
606
628
|
self.name = name
|
|
@@ -609,23 +631,23 @@ class SetIDDMSModel(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
609
631
|
def description(self) -> str:
|
|
610
632
|
return f"Sets the Data Model ID to {self.new_id.as_tuple()}"
|
|
611
633
|
|
|
612
|
-
def transform(self,
|
|
634
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
613
635
|
if self.new_id.version is None:
|
|
614
636
|
raise NeatValueError("Version is required when setting a new Data Model ID")
|
|
615
|
-
dump =
|
|
637
|
+
dump = data_model.dump()
|
|
616
638
|
dump["metadata"]["space"] = self.new_id.space
|
|
617
639
|
dump["metadata"]["external_id"] = self.new_id.external_id
|
|
618
640
|
dump["metadata"]["version"] = self.new_id.version
|
|
619
641
|
dump["metadata"]["name"] = self.name or self._generate_name()
|
|
620
642
|
# Serialize and deserialize to set the new space and external_id
|
|
621
643
|
# as the default values for the new model.
|
|
622
|
-
return
|
|
644
|
+
return PhysicalDataModel.model_validate(UnverifiedPhysicalDataModel.load(dump).dump())
|
|
623
645
|
|
|
624
646
|
def _generate_name(self) -> str:
|
|
625
647
|
return title(to_words(self.new_id.external_id))
|
|
626
648
|
|
|
627
649
|
|
|
628
|
-
class ToExtensionModel(
|
|
650
|
+
class ToExtensionModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel], ABC):
|
|
629
651
|
type_: ClassVar[str]
|
|
630
652
|
|
|
631
653
|
def __init__(self, new_model_id: DataModelIdentifier) -> None:
|
|
@@ -653,10 +675,10 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
653
675
|
self.org_name = org_name
|
|
654
676
|
self.move_connections = move_connections
|
|
655
677
|
|
|
656
|
-
def transform(self,
|
|
657
|
-
return self._to_enterprise(
|
|
678
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
679
|
+
return self._to_enterprise(data_model)
|
|
658
680
|
|
|
659
|
-
def _to_enterprise(self, reference_model:
|
|
681
|
+
def _to_enterprise(self, reference_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
660
682
|
enterprise_model = reference_model.model_copy(deep=True)
|
|
661
683
|
|
|
662
684
|
enterprise_model.metadata.name = f"{self.org_name} {self.type_} data model"
|
|
@@ -697,7 +719,9 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
697
719
|
return enterprise_model
|
|
698
720
|
|
|
699
721
|
@staticmethod
|
|
700
|
-
def _create_connection_properties(
|
|
722
|
+
def _create_connection_properties(
|
|
723
|
+
data_model: PhysicalDataModel, new_views: SheetList[PhysicalView]
|
|
724
|
+
) -> SheetList[PhysicalProperty]:
|
|
701
725
|
"""Creates a new connection property for each connection property in the reference model.
|
|
702
726
|
|
|
703
727
|
This is for example when you create an enterprise model from CogniteCore, you ensure that your
|
|
@@ -705,8 +729,8 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
705
729
|
"""
|
|
706
730
|
# Note all new news have an implements attribute that points to the original view
|
|
707
731
|
previous_by_new_view = {view.implements[0]: view.view for view in new_views if view.implements}
|
|
708
|
-
connection_properties = SheetList[
|
|
709
|
-
for prop in
|
|
732
|
+
connection_properties = SheetList[PhysicalProperty]()
|
|
733
|
+
for prop in data_model.properties:
|
|
710
734
|
if (
|
|
711
735
|
isinstance(prop.value_type, ViewEntity)
|
|
712
736
|
and prop.view in previous_by_new_view
|
|
@@ -720,23 +744,27 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
720
744
|
return connection_properties
|
|
721
745
|
|
|
722
746
|
def _create_new_views(
|
|
723
|
-
self,
|
|
724
|
-
) -> tuple[
|
|
747
|
+
self, data_model: PhysicalDataModel
|
|
748
|
+
) -> tuple[
|
|
749
|
+
SheetList[PhysicalView],
|
|
750
|
+
SheetList[PhysicalContainer],
|
|
751
|
+
SheetList[PhysicalProperty],
|
|
752
|
+
]:
|
|
725
753
|
"""Creates new views for the new model.
|
|
726
754
|
|
|
727
755
|
If the dummy property is provided, it will also create a new container for each view
|
|
728
756
|
with a single property that is the dummy property.
|
|
729
757
|
"""
|
|
730
|
-
new_views = SheetList[
|
|
731
|
-
new_containers = SheetList[
|
|
732
|
-
new_properties = SheetList[
|
|
758
|
+
new_views = SheetList[PhysicalView]()
|
|
759
|
+
new_containers = SheetList[PhysicalContainer]()
|
|
760
|
+
new_properties = SheetList[PhysicalProperty]()
|
|
733
761
|
|
|
734
|
-
for definition in
|
|
762
|
+
for definition in data_model.views:
|
|
735
763
|
view_entity = self._remove_cognite_affix(definition.view)
|
|
736
764
|
view_entity.version = cast(str, self.new_model_id.version)
|
|
737
765
|
view_entity.prefix = self.new_model_id.space
|
|
738
766
|
new_views.append(
|
|
739
|
-
|
|
767
|
+
PhysicalView(
|
|
740
768
|
view=view_entity,
|
|
741
769
|
implements=[definition.view],
|
|
742
770
|
in_model=True,
|
|
@@ -749,10 +777,10 @@ class ToEnterpriseModel(ToExtensionModel):
|
|
|
749
777
|
|
|
750
778
|
container_entity = ContainerEntity(space=view_entity.prefix, externalId=view_entity.external_id)
|
|
751
779
|
|
|
752
|
-
container =
|
|
780
|
+
container = PhysicalContainer(container=container_entity)
|
|
753
781
|
|
|
754
782
|
property_id = f"{to_camel_case(view_entity.suffix)}{self.dummy_property}"
|
|
755
|
-
property_ =
|
|
783
|
+
property_ = PhysicalProperty(
|
|
756
784
|
view=view_entity,
|
|
757
785
|
view_property=property_id,
|
|
758
786
|
value_type=String(),
|
|
@@ -810,8 +838,8 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
810
838
|
self.exclude_views_in_other_spaces = exclude_views_in_other_spaces
|
|
811
839
|
self.skip_cognite_views = skip_cognite_views
|
|
812
840
|
|
|
813
|
-
def transform(self,
|
|
814
|
-
reference_model =
|
|
841
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
842
|
+
reference_model = data_model
|
|
815
843
|
reference_model_id = reference_model.metadata.as_data_model_id()
|
|
816
844
|
if reference_model_id in COGNITE_MODELS:
|
|
817
845
|
warnings.warn(
|
|
@@ -820,13 +848,13 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
820
848
|
)
|
|
821
849
|
return self._to_solution(reference_model)
|
|
822
850
|
|
|
823
|
-
def _to_solution(self,
|
|
824
|
-
"""For creation of solution data model /
|
|
825
|
-
|
|
851
|
+
def _to_solution(self, reference_data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
852
|
+
"""For creation of solution data model / data_model specifically for mapping over existing containers."""
|
|
853
|
+
reference_data_model = self._expand_properties(reference_data_model.model_copy(deep=True))
|
|
826
854
|
|
|
827
|
-
new_views, new_properties, read_view_by_new_view = self._create_views(
|
|
855
|
+
new_views, new_properties, read_view_by_new_view = self._create_views(reference_data_model)
|
|
828
856
|
new_containers, new_container_properties = self._create_containers_update_view_filter(
|
|
829
|
-
new_views,
|
|
857
|
+
new_views, reference_data_model, read_view_by_new_view
|
|
830
858
|
)
|
|
831
859
|
new_properties.extend(new_container_properties)
|
|
832
860
|
|
|
@@ -841,7 +869,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
841
869
|
else:
|
|
842
870
|
new_properties.sort(key=lambda prop: (prop.view.external_id, prop.view_property))
|
|
843
871
|
|
|
844
|
-
metadata =
|
|
872
|
+
metadata = reference_data_model.metadata.model_copy(
|
|
845
873
|
deep=True,
|
|
846
874
|
update={
|
|
847
875
|
"space": self.new_model_id.space,
|
|
@@ -850,18 +878,18 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
850
878
|
"name": f"{self.type_} data model",
|
|
851
879
|
},
|
|
852
880
|
)
|
|
853
|
-
return
|
|
881
|
+
return PhysicalDataModel(
|
|
854
882
|
metadata=metadata,
|
|
855
883
|
properties=new_properties,
|
|
856
884
|
views=new_views,
|
|
857
885
|
containers=new_containers or None,
|
|
858
|
-
enum=
|
|
859
|
-
nodes=
|
|
886
|
+
enum=reference_data_model.enum,
|
|
887
|
+
nodes=reference_data_model.nodes,
|
|
860
888
|
)
|
|
861
889
|
|
|
862
890
|
@staticmethod
|
|
863
|
-
def _expand_properties(
|
|
864
|
-
probe =
|
|
891
|
+
def _expand_properties(data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
892
|
+
probe = DataModelAnalysis(physical=data_model)
|
|
865
893
|
ancestor_properties_by_view = probe.properties_by_view(
|
|
866
894
|
include_ancestors=True,
|
|
867
895
|
include_different_space=True,
|
|
@@ -880,15 +908,19 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
880
908
|
# original property will point to the parent view, and not the child.
|
|
881
909
|
continue
|
|
882
910
|
if prop.view_property not in property_ids:
|
|
883
|
-
|
|
911
|
+
data_model.properties.append(prop)
|
|
884
912
|
property_ids.add(prop.view_property)
|
|
885
|
-
return
|
|
913
|
+
return data_model
|
|
886
914
|
|
|
887
915
|
def _create_views(
|
|
888
|
-
self, reference:
|
|
889
|
-
) -> tuple[
|
|
916
|
+
self, reference: PhysicalDataModel
|
|
917
|
+
) -> tuple[
|
|
918
|
+
SheetList[PhysicalView],
|
|
919
|
+
SheetList[PhysicalProperty],
|
|
920
|
+
dict[ViewEntity, ViewEntity],
|
|
921
|
+
]:
|
|
890
922
|
renaming: dict[ViewEntity, ViewEntity] = {}
|
|
891
|
-
new_views = SheetList[
|
|
923
|
+
new_views = SheetList[PhysicalView]()
|
|
892
924
|
read_view_by_new_view: dict[ViewEntity, ViewEntity] = {}
|
|
893
925
|
for ref_view in reference.views:
|
|
894
926
|
if (self.skip_cognite_views and ref_view.view.space in COGNITE_SPACES) or (
|
|
@@ -906,7 +938,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
906
938
|
# This will be used to point to the one view in the Enterprise model,
|
|
907
939
|
# while the new view will to be written to.
|
|
908
940
|
new_entity.suffix = f"{self.view_prefix}{ref_view.view.suffix}"
|
|
909
|
-
new_view =
|
|
941
|
+
new_view = PhysicalView(
|
|
910
942
|
view=ViewEntity(
|
|
911
943
|
# MyPy we validate that version is string in the constructor
|
|
912
944
|
space=self.new_model_id.space,
|
|
@@ -925,7 +957,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
925
957
|
renaming[ref_view.view] = new_entity
|
|
926
958
|
new_views.append(ref_view.model_copy(deep=True, update={"implements": None, "view": new_entity}))
|
|
927
959
|
|
|
928
|
-
new_properties = SheetList[
|
|
960
|
+
new_properties = SheetList[PhysicalProperty]()
|
|
929
961
|
new_view_entities = {view.view for view in new_views}
|
|
930
962
|
for prop in reference.properties:
|
|
931
963
|
new_property = prop.model_copy(deep=True)
|
|
@@ -940,10 +972,13 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
940
972
|
return new_views, new_properties, read_view_by_new_view
|
|
941
973
|
|
|
942
974
|
def _create_containers_update_view_filter(
|
|
943
|
-
self,
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
975
|
+
self,
|
|
976
|
+
new_views: SheetList[PhysicalView],
|
|
977
|
+
reference: PhysicalDataModel,
|
|
978
|
+
read_view_by_new_view: dict[ViewEntity, ViewEntity],
|
|
979
|
+
) -> tuple[SheetList[PhysicalContainer], SheetList[PhysicalProperty]]:
|
|
980
|
+
new_containers = SheetList[PhysicalContainer]()
|
|
981
|
+
container_properties: SheetList[PhysicalProperty] = SheetList[PhysicalProperty]()
|
|
947
982
|
ref_containers_by_ref_view: dict[ViewEntity, set[ContainerEntity]] = defaultdict(set)
|
|
948
983
|
ref_views_by_external_id = {
|
|
949
984
|
view.view.external_id: view for view in reference.views if view.view.space == reference.metadata.space
|
|
@@ -959,7 +994,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
959
994
|
container_entity = ContainerEntity(space=self.new_model_id.space, externalId=view.view.external_id)
|
|
960
995
|
prefix = to_camel_case(view.view.suffix)
|
|
961
996
|
if self.properties == "repeat" and self.dummy_property:
|
|
962
|
-
property_ =
|
|
997
|
+
property_ = PhysicalProperty(
|
|
963
998
|
view=view.view,
|
|
964
999
|
view_property=f"{prefix}{self.dummy_property}",
|
|
965
1000
|
value_type=String(),
|
|
@@ -969,7 +1004,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
969
1004
|
container=container_entity,
|
|
970
1005
|
container_property=f"{prefix}{self.dummy_property}",
|
|
971
1006
|
)
|
|
972
|
-
new_containers.append(
|
|
1007
|
+
new_containers.append(PhysicalContainer(container=container_entity))
|
|
973
1008
|
container_properties.append(property_)
|
|
974
1009
|
elif self.properties == "repeat" and self.dummy_property is None:
|
|
975
1010
|
# For this case we set the filter. This is used by the DataProductModel.
|
|
@@ -978,7 +1013,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
978
1013
|
if ref_view := ref_views_by_external_id.get(view.view.external_id):
|
|
979
1014
|
self._set_view_filter(view, ref_containers_by_ref_view, ref_view)
|
|
980
1015
|
elif self.properties == "connection" and self.direct_property:
|
|
981
|
-
property_ =
|
|
1016
|
+
property_ = PhysicalProperty(
|
|
982
1017
|
view=view.view,
|
|
983
1018
|
view_property=self.direct_property,
|
|
984
1019
|
value_type=read_view,
|
|
@@ -989,7 +1024,7 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
989
1024
|
container_property=self.direct_property,
|
|
990
1025
|
connection="direct",
|
|
991
1026
|
)
|
|
992
|
-
new_containers.append(
|
|
1027
|
+
new_containers.append(PhysicalContainer(container=container_entity))
|
|
993
1028
|
container_properties.append(property_)
|
|
994
1029
|
else:
|
|
995
1030
|
raise NeatValueError(f"Unsupported properties mode: {self.properties}")
|
|
@@ -1002,7 +1037,10 @@ class ToSolutionModel(ToExtensionModel):
|
|
|
1002
1037
|
return new_containers, container_properties
|
|
1003
1038
|
|
|
1004
1039
|
def _set_view_filter(
|
|
1005
|
-
self,
|
|
1040
|
+
self,
|
|
1041
|
+
view: PhysicalView,
|
|
1042
|
+
ref_containers_by_ref_view: dict[ViewEntity, set[ContainerEntity]],
|
|
1043
|
+
ref_view: PhysicalView,
|
|
1006
1044
|
) -> None:
|
|
1007
1045
|
if self.filter_type == "view":
|
|
1008
1046
|
view.filter_ = HasDataFilter(inner=[ref_view.view])
|
|
@@ -1042,12 +1080,12 @@ class ToDataProductModel(ToSolutionModel):
|
|
|
1042
1080
|
)
|
|
1043
1081
|
self.include = include
|
|
1044
1082
|
|
|
1045
|
-
def transform(self,
|
|
1083
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1046
1084
|
# Overwrite transform to avoid the warning.
|
|
1047
|
-
return self._to_solution(
|
|
1085
|
+
return self._to_solution(data_model)
|
|
1048
1086
|
|
|
1049
1087
|
|
|
1050
|
-
class DropModelViews(
|
|
1088
|
+
class DropModelViews(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1051
1089
|
_ASSET_VIEW = ViewId("cdf_cdm", "CogniteAsset", "v1")
|
|
1052
1090
|
_VIEW_BY_COLLECTION: Mapping[Literal["3D", "Annotation", "BaseViews"], frozenset[ViewId]] = {
|
|
1053
1091
|
"3D": frozenset(
|
|
@@ -1105,22 +1143,22 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1105
1143
|
)
|
|
1106
1144
|
)
|
|
1107
1145
|
|
|
1108
|
-
def transform(self,
|
|
1146
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1109
1147
|
exclude_views: set[ViewEntity] = {
|
|
1110
|
-
view.view for view in
|
|
1148
|
+
view.view for view in data_model.views if view.view.suffix in self.drop_external_ids
|
|
1111
1149
|
}
|
|
1112
|
-
if
|
|
1150
|
+
if data_model.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
1113
1151
|
exclude_views |= {
|
|
1114
1152
|
ViewEntity.from_id(view_id, "v1")
|
|
1115
1153
|
for collection in self.drop_collection
|
|
1116
1154
|
for view_id in self._VIEW_BY_COLLECTION[collection]
|
|
1117
1155
|
}
|
|
1118
|
-
new_model =
|
|
1156
|
+
new_model = data_model.model_copy(deep=True)
|
|
1119
1157
|
|
|
1120
|
-
properties_by_view =
|
|
1158
|
+
properties_by_view = DataModelAnalysis(physical=new_model).properties_by_view(include_ancestors=True)
|
|
1121
1159
|
|
|
1122
|
-
new_model.views = SheetList[
|
|
1123
|
-
new_properties = SheetList[
|
|
1160
|
+
new_model.views = SheetList[PhysicalView]([view for view in new_model.views if view.view not in exclude_views])
|
|
1161
|
+
new_properties = SheetList[PhysicalProperty]()
|
|
1124
1162
|
mapped_containers: set[ContainerEntity] = set()
|
|
1125
1163
|
for view in new_model.views:
|
|
1126
1164
|
for prop in properties_by_view[view.view]:
|
|
@@ -1135,7 +1173,7 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1135
1173
|
|
|
1136
1174
|
new_model.properties = new_properties
|
|
1137
1175
|
new_model.containers = (
|
|
1138
|
-
SheetList[
|
|
1176
|
+
SheetList[PhysicalContainer](
|
|
1139
1177
|
[container for container in new_model.containers or [] if container.container in mapped_containers]
|
|
1140
1178
|
)
|
|
1141
1179
|
or None
|
|
@@ -1144,7 +1182,7 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1144
1182
|
return new_model
|
|
1145
1183
|
|
|
1146
1184
|
@classmethod
|
|
1147
|
-
def _is_asset_3D_property(cls, prop:
|
|
1185
|
+
def _is_asset_3D_property(cls, prop: PhysicalProperty) -> bool:
|
|
1148
1186
|
return prop.view.as_id() == cls._ASSET_VIEW and prop.view_property == "object3D"
|
|
1149
1187
|
|
|
1150
1188
|
@property
|
|
@@ -1152,37 +1190,37 @@ class DropModelViews(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1152
1190
|
return f"Removed {len(self.drop_external_ids) + len(self.drop_collection)} views from data model"
|
|
1153
1191
|
|
|
1154
1192
|
|
|
1155
|
-
class IncludeReferenced(
|
|
1193
|
+
class IncludeReferenced(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1156
1194
|
def __init__(self, client: NeatClient, include_properties: bool = False) -> None:
|
|
1157
1195
|
self._client = client
|
|
1158
1196
|
self.include_properties = include_properties
|
|
1159
1197
|
|
|
1160
|
-
def transform(self,
|
|
1161
|
-
|
|
1162
|
-
view_ids, container_ids =
|
|
1198
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1199
|
+
physical_data_model = data_model
|
|
1200
|
+
view_ids, container_ids = PhysicalValidation(physical_data_model).imported_views_and_containers_ids()
|
|
1163
1201
|
if not (view_ids or container_ids):
|
|
1164
1202
|
warnings.warn(
|
|
1165
1203
|
NeatValueWarning(
|
|
1166
|
-
f"Data model {
|
|
1204
|
+
f"Data model {physical_data_model.metadata.as_data_model_id()} does not have any "
|
|
1167
1205
|
"referenced views or containers."
|
|
1168
1206
|
"that is not already included in the data model."
|
|
1169
1207
|
),
|
|
1170
1208
|
stacklevel=2,
|
|
1171
1209
|
)
|
|
1172
|
-
return
|
|
1210
|
+
return physical_data_model
|
|
1173
1211
|
|
|
1174
1212
|
schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
|
|
1175
|
-
copy_ =
|
|
1213
|
+
copy_ = physical_data_model.model_copy(deep=True)
|
|
1176
1214
|
# Sorting to ensure deterministic order
|
|
1177
1215
|
schema.containers = ContainerApplyDict(sorted(schema.containers.items(), key=lambda x: x[0].as_tuple()))
|
|
1178
1216
|
schema.views = ViewApplyDict(sorted(schema.views.items(), key=lambda x: x[0].as_tuple()))
|
|
1179
1217
|
importer = DMSImporter(schema)
|
|
1180
1218
|
|
|
1181
|
-
imported = importer.
|
|
1182
|
-
if imported.
|
|
1219
|
+
imported = importer.to_data_model()
|
|
1220
|
+
if imported.unverified_data_model is None:
|
|
1183
1221
|
raise NeatValueError("Could not import the referenced views and containers.")
|
|
1184
1222
|
|
|
1185
|
-
verified =
|
|
1223
|
+
verified = VerifyPhysicalDataModel(validate=False).transform(imported)
|
|
1186
1224
|
if copy_.containers is None:
|
|
1187
1225
|
copy_.containers = verified.containers
|
|
1188
1226
|
else:
|
|
@@ -1203,17 +1241,16 @@ class IncludeReferenced(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1203
1241
|
return "Included referenced views and containers in the data model."
|
|
1204
1242
|
|
|
1205
1243
|
|
|
1206
|
-
class
|
|
1244
|
+
class AddConceptImplements(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
1207
1245
|
def __init__(self, implements: str, suffix: str):
|
|
1208
1246
|
self.implements = implements
|
|
1209
1247
|
self.suffix = suffix
|
|
1210
1248
|
|
|
1211
|
-
def transform(self,
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
class_.implements = [ClassEntity(prefix=class_.class_.prefix, suffix=self.implements)]
|
|
1249
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
1250
|
+
output = data_model.model_copy(deep=True)
|
|
1251
|
+
for concept in output.concepts:
|
|
1252
|
+
if concept.concept.suffix.endswith(self.suffix):
|
|
1253
|
+
concept.implements = [ConceptEntity(prefix=concept.concept.prefix, suffix=self.implements)]
|
|
1217
1254
|
return output
|
|
1218
1255
|
|
|
1219
1256
|
@property
|
|
@@ -1221,7 +1258,7 @@ class AddClassImplements(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1221
1258
|
return f"Added implements property to classes with suffix {self.suffix}"
|
|
1222
1259
|
|
|
1223
1260
|
|
|
1224
|
-
class ClassicPrepareCore(
|
|
1261
|
+
class ClassicPrepareCore(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
1225
1262
|
"""Update the classic data model with the following:
|
|
1226
1263
|
|
|
1227
1264
|
This is a special purpose transformer that is only intended to be used with when reading
|
|
@@ -1250,32 +1287,32 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1250
1287
|
def description(self) -> str:
|
|
1251
1288
|
return "Update the classic data model to the data types in Cognite Core."
|
|
1252
1289
|
|
|
1253
|
-
def transform(self,
|
|
1254
|
-
output =
|
|
1290
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
1291
|
+
output = data_model.model_copy(deep=True)
|
|
1255
1292
|
for prop in output.properties:
|
|
1256
|
-
if prop.
|
|
1293
|
+
if prop.concept.suffix == "Timeseries" and prop.property_ == "isString":
|
|
1257
1294
|
prop.value_type = String()
|
|
1258
1295
|
prefix = output.metadata.prefix
|
|
1259
1296
|
namespace = output.metadata.namespace
|
|
1260
|
-
source_system_class =
|
|
1261
|
-
|
|
1297
|
+
source_system_class = Concept(
|
|
1298
|
+
concept=ConceptEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1262
1299
|
description="A source system that provides data to the data model.",
|
|
1263
1300
|
neatId=namespace["ClassicSourceSystem"],
|
|
1264
1301
|
instance_source=self.instance_namespace["ClassicSourceSystem"],
|
|
1265
1302
|
)
|
|
1266
|
-
output.
|
|
1303
|
+
output.concepts.append(source_system_class)
|
|
1267
1304
|
for prop in output.properties:
|
|
1268
|
-
if prop.property_ == "source" and prop.
|
|
1269
|
-
prop.value_type =
|
|
1305
|
+
if prop.property_ == "source" and prop.concept.suffix != "ClassicSourceSystem":
|
|
1306
|
+
prop.value_type = ConceptEntity(prefix=prefix, suffix="ClassicSourceSystem")
|
|
1270
1307
|
elif prop.property_ == "externalId":
|
|
1271
1308
|
prop.property_ = "classicExternalId"
|
|
1272
|
-
if self.reference_timeseries and prop.
|
|
1309
|
+
if self.reference_timeseries and prop.concept.suffix == "ClassicTimeSeries":
|
|
1273
1310
|
prop.value_type = Timeseries()
|
|
1274
|
-
elif self.reference_files and prop.
|
|
1311
|
+
elif self.reference_files and prop.concept.suffix == "ClassicFile":
|
|
1275
1312
|
prop.value_type = File()
|
|
1276
|
-
elif prop.property_ == "sourceExternalId" and prop.
|
|
1313
|
+
elif prop.property_ == "sourceExternalId" and prop.concept.suffix == "ClassicRelationship":
|
|
1277
1314
|
prop.property_ = "startNode"
|
|
1278
|
-
elif prop.property_ == "targetExternalId" and prop.
|
|
1315
|
+
elif prop.property_ == "targetExternalId" and prop.concept.suffix == "ClassicRelationship":
|
|
1279
1316
|
prop.property_ = "endNode"
|
|
1280
1317
|
instance_prefix = next(
|
|
1281
1318
|
(prefix for prefix, namespace in output.prefixes.items() if namespace == self.instance_namespace), None
|
|
@@ -1284,11 +1321,11 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1284
1321
|
raise NeatValueError("Instance namespace not found in the prefixes.")
|
|
1285
1322
|
|
|
1286
1323
|
output.properties.append(
|
|
1287
|
-
|
|
1324
|
+
ConceptualProperty(
|
|
1288
1325
|
neatId=namespace["ClassicSourceSystem/name"],
|
|
1289
1326
|
property_="name",
|
|
1290
1327
|
value_type=String(),
|
|
1291
|
-
|
|
1328
|
+
concept=ConceptEntity(prefix=prefix, suffix="ClassicSourceSystem"),
|
|
1292
1329
|
max_count=1,
|
|
1293
1330
|
instance_source=[self.instance_namespace["name"]],
|
|
1294
1331
|
)
|
|
@@ -1296,13 +1333,13 @@ class ClassicPrepareCore(VerifiedRulesTransformer[InformationRules, InformationR
|
|
|
1296
1333
|
return output
|
|
1297
1334
|
|
|
1298
1335
|
|
|
1299
|
-
class ChangeViewPrefix(
|
|
1336
|
+
class ChangeViewPrefix(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1300
1337
|
def __init__(self, old: str, new: str) -> None:
|
|
1301
1338
|
self.old = old
|
|
1302
1339
|
self.new = new
|
|
1303
1340
|
|
|
1304
|
-
def transform(self,
|
|
1305
|
-
output =
|
|
1341
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1342
|
+
output = data_model.model_copy(deep=True)
|
|
1306
1343
|
new_by_old: dict[ViewEntity, ViewEntity] = {}
|
|
1307
1344
|
for view in output.views:
|
|
1308
1345
|
if view.view.external_id.startswith(self.old):
|
|
@@ -1321,12 +1358,12 @@ class ChangeViewPrefix(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1321
1358
|
return output
|
|
1322
1359
|
|
|
1323
1360
|
|
|
1324
|
-
class
|
|
1325
|
-
def __init__(self, extra:
|
|
1361
|
+
class MergePhysicalDataModels(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
1362
|
+
def __init__(self, extra: PhysicalDataModel) -> None:
|
|
1326
1363
|
self.extra = extra
|
|
1327
1364
|
|
|
1328
|
-
def transform(self,
|
|
1329
|
-
output =
|
|
1365
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
1366
|
+
output = data_model.model_copy(deep=True)
|
|
1330
1367
|
existing_views = {view.view for view in output.views}
|
|
1331
1368
|
for view in self.extra.views:
|
|
1332
1369
|
if view.view not in existing_views:
|
|
@@ -1342,18 +1379,18 @@ class MergeDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1342
1379
|
output.properties.append(prop)
|
|
1343
1380
|
if prop.container and prop.container not in existing_containers:
|
|
1344
1381
|
if output.containers is None:
|
|
1345
|
-
output.containers = SheetList[
|
|
1382
|
+
output.containers = SheetList[PhysicalContainer]()
|
|
1346
1383
|
output.containers.append(new_containers_by_entity[prop.container])
|
|
1347
1384
|
if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections:
|
|
1348
1385
|
if output.enum is None:
|
|
1349
|
-
output.enum = SheetList[
|
|
1386
|
+
output.enum = SheetList[PhysicalEnum]()
|
|
1350
1387
|
output.enum.append(new_enum_collections_by_entity[prop.value_type.collection])
|
|
1351
1388
|
|
|
1352
1389
|
existing_nodes = {node.node for node in output.nodes or []}
|
|
1353
1390
|
for node in self.extra.nodes or []:
|
|
1354
1391
|
if node.node not in existing_nodes:
|
|
1355
1392
|
if output.nodes is None:
|
|
1356
|
-
output.nodes = SheetList[
|
|
1393
|
+
output.nodes = SheetList[PhysicalNodeType]()
|
|
1357
1394
|
output.nodes.append(node)
|
|
1358
1395
|
|
|
1359
1396
|
return output
|
|
@@ -1363,19 +1400,19 @@ class MergeDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
1363
1400
|
return f"Merged with {self.extra.metadata.as_data_model_id()}"
|
|
1364
1401
|
|
|
1365
1402
|
|
|
1366
|
-
class
|
|
1367
|
-
def __init__(self, extra:
|
|
1403
|
+
class MergeConceptualDataModels(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
1404
|
+
def __init__(self, extra: ConceptualDataModel) -> None:
|
|
1368
1405
|
self.extra = extra
|
|
1369
1406
|
|
|
1370
|
-
def transform(self,
|
|
1371
|
-
output =
|
|
1372
|
-
existing_classes = {cls.
|
|
1373
|
-
for cls in self.extra.
|
|
1374
|
-
if cls.
|
|
1375
|
-
output.
|
|
1376
|
-
existing_properties = {(prop.
|
|
1407
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
1408
|
+
output = data_model.model_copy(deep=True)
|
|
1409
|
+
existing_classes = {cls.concept for cls in output.concepts}
|
|
1410
|
+
for cls in self.extra.concepts:
|
|
1411
|
+
if cls.concept not in existing_classes:
|
|
1412
|
+
output.concepts.append(cls)
|
|
1413
|
+
existing_properties = {(prop.concept, prop.property_) for prop in output.properties}
|
|
1377
1414
|
for prop in self.extra.properties:
|
|
1378
|
-
if (prop.
|
|
1415
|
+
if (prop.concept, prop.property_) not in existing_properties:
|
|
1379
1416
|
output.properties.append(prop)
|
|
1380
1417
|
for prefix, namespace in self.extra.prefixes.items():
|
|
1381
1418
|
if prefix not in output.prefixes:
|
|
@@ -1383,57 +1420,63 @@ class MergeInformationRules(VerifiedRulesTransformer[InformationRules, Informati
|
|
|
1383
1420
|
return output
|
|
1384
1421
|
|
|
1385
1422
|
|
|
1386
|
-
class
|
|
1423
|
+
class _ConceptualDataModelConverter:
|
|
1387
1424
|
_start_or_end_node: ClassVar[frozenset[str]] = frozenset({"endNode", "end_node", "startNode", "start_node"})
|
|
1388
1425
|
|
|
1389
|
-
def __init__(
|
|
1390
|
-
self
|
|
1426
|
+
def __init__(
|
|
1427
|
+
self,
|
|
1428
|
+
conceptual_data_model: ConceptualDataModel,
|
|
1429
|
+
client: NeatClient | None = None,
|
|
1430
|
+
):
|
|
1431
|
+
self.conceptual_data_model = conceptual_data_model
|
|
1391
1432
|
self.property_count_by_container: dict[ContainerEntity, int] = defaultdict(int)
|
|
1392
1433
|
self.client = client
|
|
1393
1434
|
|
|
1394
|
-
def
|
|
1395
|
-
self,
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1435
|
+
def as_physical_data_model(
|
|
1436
|
+
self,
|
|
1437
|
+
ignore_undefined_value_types: bool = False,
|
|
1438
|
+
reserved_properties: Literal["error", "warning"] = "error",
|
|
1439
|
+
) -> "PhysicalDataModel":
|
|
1440
|
+
from cognite.neat.core._data_model.models.physical._verified import (
|
|
1441
|
+
PhysicalContainer,
|
|
1442
|
+
PhysicalDataModel,
|
|
1443
|
+
PhysicalProperty,
|
|
1444
|
+
PhysicalView,
|
|
1402
1445
|
)
|
|
1403
1446
|
|
|
1404
|
-
|
|
1405
|
-
default_version =
|
|
1406
|
-
default_space = self._to_space(
|
|
1407
|
-
|
|
1447
|
+
conceptual_metadata = self.conceptual_data_model.metadata
|
|
1448
|
+
default_version = conceptual_metadata.version
|
|
1449
|
+
default_space = self._to_space(conceptual_metadata.prefix)
|
|
1450
|
+
physical_metadata = self._convert_conceptual_to_physical_metadata(conceptual_metadata)
|
|
1408
1451
|
|
|
1409
|
-
|
|
1410
|
-
for prop in self.
|
|
1411
|
-
|
|
1452
|
+
properties_by_concept: dict[ConceptEntity, set[str]] = defaultdict(set)
|
|
1453
|
+
for prop in self.conceptual_data_model.properties:
|
|
1454
|
+
properties_by_concept[prop.concept].add(prop.property_)
|
|
1412
1455
|
|
|
1413
1456
|
# Edge Classes is defined by having both startNode and endNode properties
|
|
1414
1457
|
edge_classes = {
|
|
1415
|
-
|
|
1416
|
-
for
|
|
1417
|
-
if ({"startNode", "start_node"} &
|
|
1458
|
+
concept
|
|
1459
|
+
for concept, concept_properties in properties_by_concept.items()
|
|
1460
|
+
if ({"startNode", "start_node"} & concept_properties) and ({"endNode", "end_node"} & concept_properties)
|
|
1418
1461
|
}
|
|
1419
|
-
|
|
1420
|
-
(prop.
|
|
1421
|
-
for prop in self.
|
|
1422
|
-
if prop.value_type in edge_classes and isinstance(prop.value_type,
|
|
1462
|
+
edge_value_types_by_concept_property_pair = {
|
|
1463
|
+
(prop.concept, prop.property_): prop.value_type
|
|
1464
|
+
for prop in self.conceptual_data_model.properties
|
|
1465
|
+
if prop.value_type in edge_classes and isinstance(prop.value_type, ConceptEntity)
|
|
1423
1466
|
}
|
|
1424
1467
|
end_node_by_edge = {
|
|
1425
|
-
prop.
|
|
1426
|
-
for prop in self.
|
|
1427
|
-
if prop.
|
|
1468
|
+
prop.concept: prop.value_type
|
|
1469
|
+
for prop in self.conceptual_data_model.properties
|
|
1470
|
+
if prop.concept in edge_classes
|
|
1428
1471
|
and (prop.property_ == "endNode" or prop.property_ == "end_node")
|
|
1429
|
-
and isinstance(prop.value_type,
|
|
1472
|
+
and isinstance(prop.value_type, ConceptEntity)
|
|
1430
1473
|
}
|
|
1431
1474
|
ancestors_by_view: dict[ViewEntity, set[ViewEntity]] = {}
|
|
1432
|
-
|
|
1475
|
+
parents_by_concept = DataModelAnalysis(self.conceptual_data_model).parents_by_concept(
|
|
1433
1476
|
include_ancestors=True, include_different_space=True
|
|
1434
1477
|
)
|
|
1435
|
-
for
|
|
1436
|
-
view_type =
|
|
1478
|
+
for concept, parents in parents_by_concept.items():
|
|
1479
|
+
view_type = concept.as_view_entity(default_space, default_version)
|
|
1437
1480
|
parent_views = {parent.as_view_entity(default_space, default_version) for parent in parents}
|
|
1438
1481
|
if view_type in ancestors_by_view:
|
|
1439
1482
|
ancestors_by_view[view_type].update(parent_views)
|
|
@@ -1447,14 +1490,14 @@ class _InformationRulesConverter:
|
|
|
1447
1490
|
else:
|
|
1448
1491
|
ancestors_by_view[view] = cognite_views[view]
|
|
1449
1492
|
|
|
1450
|
-
|
|
1451
|
-
used_containers: dict[ContainerEntity, Counter[
|
|
1452
|
-
used_cognite_containers: dict[ContainerEntity,
|
|
1493
|
+
properties_by_concept: dict[ConceptEntity, list[PhysicalProperty]] = defaultdict(list)
|
|
1494
|
+
used_containers: dict[ContainerEntity, Counter[ConceptEntity]] = defaultdict(Counter)
|
|
1495
|
+
used_cognite_containers: dict[ContainerEntity, PhysicalContainer] = {}
|
|
1453
1496
|
|
|
1454
|
-
for prop in self.
|
|
1497
|
+
for prop in self.conceptual_data_model.properties:
|
|
1455
1498
|
if ignore_undefined_value_types and isinstance(prop.value_type, UnknownEntity):
|
|
1456
1499
|
continue
|
|
1457
|
-
if prop.
|
|
1500
|
+
if prop.concept in edge_classes and prop.property_ in self._start_or_end_node:
|
|
1458
1501
|
continue
|
|
1459
1502
|
if prop.property_ in DMS_RESERVED_PROPERTIES:
|
|
1460
1503
|
msg = f"Property {prop.property_} is a reserved property in DMS."
|
|
@@ -1464,72 +1507,77 @@ class _InformationRulesConverter:
|
|
|
1464
1507
|
continue
|
|
1465
1508
|
|
|
1466
1509
|
if cognite_property := self._find_cognite_property(
|
|
1467
|
-
prop.property_,
|
|
1510
|
+
prop.property_, parents_by_concept[prop.concept], cognite_properties
|
|
1468
1511
|
):
|
|
1469
|
-
|
|
1512
|
+
physical_property = self._customize_physical_property(
|
|
1470
1513
|
prop,
|
|
1471
1514
|
cognite_property,
|
|
1472
|
-
prop.
|
|
1515
|
+
prop.concept,
|
|
1473
1516
|
default_space,
|
|
1474
1517
|
default_version,
|
|
1475
1518
|
ancestors_by_view,
|
|
1476
1519
|
)
|
|
1477
|
-
if
|
|
1478
|
-
if
|
|
1479
|
-
used_cognite_containers[
|
|
1520
|
+
if physical_property.container:
|
|
1521
|
+
if physical_property.container not in used_cognite_containers:
|
|
1522
|
+
used_cognite_containers[physical_property.container] = cognite_containers[
|
|
1523
|
+
physical_property.container
|
|
1524
|
+
]
|
|
1480
1525
|
else:
|
|
1481
1526
|
# Not matching any parent.
|
|
1482
|
-
|
|
1527
|
+
physical_property = self._as_physical_property(
|
|
1483
1528
|
prop,
|
|
1484
1529
|
default_space,
|
|
1485
1530
|
default_version,
|
|
1486
1531
|
edge_classes,
|
|
1487
|
-
|
|
1532
|
+
edge_value_types_by_concept_property_pair,
|
|
1488
1533
|
end_node_by_edge,
|
|
1489
1534
|
)
|
|
1490
|
-
if
|
|
1491
|
-
used_containers[
|
|
1535
|
+
if physical_property.container:
|
|
1536
|
+
used_containers[physical_property.container][prop.concept] += 1
|
|
1492
1537
|
|
|
1493
|
-
|
|
1538
|
+
properties_by_concept[prop.concept].append(physical_property)
|
|
1494
1539
|
|
|
1495
|
-
views: list[
|
|
1540
|
+
views: list[PhysicalView] = []
|
|
1496
1541
|
|
|
1497
|
-
for
|
|
1498
|
-
|
|
1499
|
-
name=
|
|
1500
|
-
view=
|
|
1501
|
-
description=
|
|
1502
|
-
implements=self._get_view_implements(
|
|
1542
|
+
for concept in self.conceptual_data_model.concepts:
|
|
1543
|
+
physical_view = PhysicalView(
|
|
1544
|
+
name=concept.name,
|
|
1545
|
+
view=concept.concept.as_view_entity(default_space, default_version),
|
|
1546
|
+
description=concept.description,
|
|
1547
|
+
implements=self._get_view_implements(concept, conceptual_metadata),
|
|
1503
1548
|
)
|
|
1504
1549
|
|
|
1505
|
-
|
|
1506
|
-
views.append(
|
|
1550
|
+
physical_view.conceptual = concept.neatId
|
|
1551
|
+
views.append(physical_view)
|
|
1507
1552
|
|
|
1508
|
-
|
|
1553
|
+
concept_by_concept_entity = {cls_.concept: cls_ for cls_ in self.conceptual_data_model.concepts}
|
|
1509
1554
|
|
|
1510
1555
|
existing_containers: set[ContainerEntity] = set()
|
|
1511
1556
|
|
|
1512
|
-
containers: list[
|
|
1513
|
-
for container_entity,
|
|
1557
|
+
containers: list[PhysicalContainer] = []
|
|
1558
|
+
for container_entity, concept_entities in used_containers.items():
|
|
1514
1559
|
if container_entity in existing_containers:
|
|
1515
1560
|
continue
|
|
1516
1561
|
constrains = self._create_container_constraint(
|
|
1517
|
-
|
|
1562
|
+
concept_entities,
|
|
1563
|
+
default_space,
|
|
1564
|
+
concept_by_concept_entity,
|
|
1565
|
+
used_containers,
|
|
1518
1566
|
)
|
|
1519
|
-
|
|
1520
|
-
|
|
1567
|
+
most_used_concept_entity = concept_entities.most_common(1)[0][0]
|
|
1568
|
+
concept = concept_by_concept_entity[most_used_concept_entity]
|
|
1521
1569
|
|
|
1522
|
-
if len(set(
|
|
1570
|
+
if len(set(concept_entities) - set(edge_classes)) == 0:
|
|
1523
1571
|
used_for: Literal["node", "edge", "all"] = "edge"
|
|
1524
|
-
elif len(set(
|
|
1572
|
+
elif len(set(concept_entities) - set(edge_classes)) == len(concept_entities):
|
|
1525
1573
|
used_for = "node"
|
|
1526
1574
|
else:
|
|
1527
1575
|
used_for = "all"
|
|
1528
1576
|
|
|
1529
|
-
container =
|
|
1577
|
+
container = PhysicalContainer(
|
|
1530
1578
|
container=container_entity,
|
|
1531
|
-
name=
|
|
1532
|
-
description=
|
|
1579
|
+
name=concept.name,
|
|
1580
|
+
description=concept.description,
|
|
1533
1581
|
constraint=constrains or None,
|
|
1534
1582
|
used_for=used_for,
|
|
1535
1583
|
)
|
|
@@ -1538,69 +1586,75 @@ class _InformationRulesConverter:
|
|
|
1538
1586
|
if used_cognite_containers:
|
|
1539
1587
|
containers.extend(used_cognite_containers.values())
|
|
1540
1588
|
|
|
1541
|
-
|
|
1542
|
-
metadata=
|
|
1543
|
-
properties=SheetList[
|
|
1544
|
-
|
|
1545
|
-
|
|
1589
|
+
physical_data_model = PhysicalDataModel(
|
|
1590
|
+
metadata=physical_metadata,
|
|
1591
|
+
properties=SheetList[PhysicalProperty](
|
|
1592
|
+
[prop for prop_set in properties_by_concept.values() for prop in prop_set]
|
|
1593
|
+
),
|
|
1594
|
+
views=SheetList[PhysicalView](views),
|
|
1595
|
+
containers=SheetList[PhysicalContainer](containers),
|
|
1546
1596
|
)
|
|
1547
1597
|
|
|
1548
|
-
self.
|
|
1598
|
+
self.conceptual_data_model.sync_with_physical_data_model(physical_data_model)
|
|
1549
1599
|
|
|
1550
|
-
return
|
|
1600
|
+
return physical_data_model
|
|
1551
1601
|
|
|
1552
1602
|
def _get_cognite_components(
|
|
1553
1603
|
self,
|
|
1554
1604
|
) -> tuple[
|
|
1555
|
-
dict[tuple[
|
|
1556
|
-
dict[ContainerEntity,
|
|
1605
|
+
dict[tuple[ConceptEntity, str], PhysicalProperty],
|
|
1606
|
+
dict[ContainerEntity, PhysicalContainer],
|
|
1557
1607
|
dict[ViewEntity, set[ViewEntity]],
|
|
1558
1608
|
]:
|
|
1559
1609
|
cognite_concepts = self._get_cognite_concepts()
|
|
1560
|
-
cognite_properties: dict[tuple[
|
|
1561
|
-
cognite_containers: dict[ContainerEntity,
|
|
1610
|
+
cognite_properties: dict[tuple[ConceptEntity, str], PhysicalProperty] = {}
|
|
1611
|
+
cognite_containers: dict[ContainerEntity, PhysicalContainer] = {}
|
|
1562
1612
|
cognite_views: dict[ViewEntity, set[ViewEntity]] = {}
|
|
1563
1613
|
if cognite_concepts:
|
|
1564
1614
|
if self.client is None:
|
|
1565
1615
|
raise CDFMissingClientError(
|
|
1566
|
-
f"Cannot convert {self.
|
|
1616
|
+
f"Cannot convert {self.conceptual_data_model.metadata.as_data_model_id()}. Missing Cognite Client."
|
|
1567
1617
|
f"This is required as the data model is referencing cognite concepts in the implements"
|
|
1568
1618
|
f"{humanize_collection(cognite_concepts)}"
|
|
1569
1619
|
)
|
|
1570
|
-
|
|
1620
|
+
cognite_data_model = self._get_cognite_physical_data_model(cognite_concepts, self.client)
|
|
1571
1621
|
|
|
1572
1622
|
cognite_properties = {
|
|
1573
|
-
(
|
|
1623
|
+
(
|
|
1624
|
+
physical_property.view.as_concept_entity(),
|
|
1625
|
+
physical_property.view_property,
|
|
1626
|
+
): physical_property
|
|
1627
|
+
for physical_property in cognite_data_model.properties
|
|
1574
1628
|
}
|
|
1575
|
-
cognite_containers = {container.container: container for container in
|
|
1576
|
-
cognite_views =
|
|
1629
|
+
cognite_containers = {container.container: container for container in cognite_data_model.containers or []}
|
|
1630
|
+
cognite_views = DataModelAnalysis(physical=cognite_data_model).implements_by_view(
|
|
1577
1631
|
include_ancestors=True, include_different_space=True
|
|
1578
1632
|
)
|
|
1579
1633
|
return cognite_properties, cognite_containers, cognite_views
|
|
1580
1634
|
|
|
1581
1635
|
@staticmethod
|
|
1582
1636
|
def _create_container_constraint(
|
|
1583
|
-
|
|
1637
|
+
concept_entities: Counter[ConceptEntity],
|
|
1584
1638
|
default_space: str,
|
|
1585
|
-
|
|
1639
|
+
concept_by_concept_entity: dict[ConceptEntity, Concept],
|
|
1586
1640
|
referenced_containers: Collection[ContainerEntity],
|
|
1587
1641
|
) -> list[ContainerEntity]:
|
|
1588
1642
|
constrains: list[ContainerEntity] = []
|
|
1589
|
-
for entity in
|
|
1590
|
-
|
|
1591
|
-
for parent in
|
|
1643
|
+
for entity in concept_entities:
|
|
1644
|
+
concept = concept_by_concept_entity[entity]
|
|
1645
|
+
for parent in concept.implements or []:
|
|
1592
1646
|
parent_entity = parent.as_container_entity(default_space)
|
|
1593
1647
|
if parent_entity in referenced_containers:
|
|
1594
1648
|
constrains.append(parent_entity)
|
|
1595
1649
|
return constrains
|
|
1596
1650
|
|
|
1597
1651
|
@classmethod
|
|
1598
|
-
def
|
|
1599
|
-
from cognite.neat.core.
|
|
1600
|
-
|
|
1652
|
+
def _convert_conceptual_to_physical_metadata(cls, metadata: ConceptualMetadata) -> "PhysicalMetadata":
|
|
1653
|
+
from cognite.neat.core._data_model.models.physical._verified import (
|
|
1654
|
+
PhysicalMetadata,
|
|
1601
1655
|
)
|
|
1602
1656
|
|
|
1603
|
-
|
|
1657
|
+
physical_metadata = PhysicalMetadata(
|
|
1604
1658
|
space=metadata.space,
|
|
1605
1659
|
version=metadata.version,
|
|
1606
1660
|
external_id=metadata.external_id,
|
|
@@ -1610,23 +1664,23 @@ class _InformationRulesConverter:
|
|
|
1610
1664
|
updated=metadata.updated,
|
|
1611
1665
|
)
|
|
1612
1666
|
|
|
1613
|
-
|
|
1614
|
-
return
|
|
1667
|
+
physical_metadata.conceptual = metadata.identifier
|
|
1668
|
+
return physical_metadata
|
|
1615
1669
|
|
|
1616
|
-
def
|
|
1670
|
+
def _as_physical_property(
|
|
1617
1671
|
self,
|
|
1618
|
-
|
|
1672
|
+
conceptual_property: ConceptualProperty,
|
|
1619
1673
|
default_space: str,
|
|
1620
1674
|
default_version: str,
|
|
1621
|
-
edge_classes: set[
|
|
1622
|
-
|
|
1623
|
-
end_node_by_edge: dict[
|
|
1624
|
-
) -> "
|
|
1625
|
-
from cognite.neat.core.
|
|
1675
|
+
edge_classes: set[ConceptEntity],
|
|
1676
|
+
edge_value_types_by_concept_property_pair: dict[tuple[ConceptEntity, str], ConceptEntity],
|
|
1677
|
+
end_node_by_edge: dict[ConceptEntity, ConceptEntity],
|
|
1678
|
+
) -> "PhysicalProperty":
|
|
1679
|
+
from cognite.neat.core._data_model.models.physical._verified import PhysicalProperty
|
|
1626
1680
|
|
|
1627
1681
|
# returns property type, which can be ObjectProperty or DatatypeProperty
|
|
1628
1682
|
value_type = self._get_value_type(
|
|
1629
|
-
|
|
1683
|
+
conceptual_property,
|
|
1630
1684
|
default_space,
|
|
1631
1685
|
default_version,
|
|
1632
1686
|
edge_classes,
|
|
@@ -1634,14 +1688,18 @@ class _InformationRulesConverter:
|
|
|
1634
1688
|
)
|
|
1635
1689
|
|
|
1636
1690
|
connection = self._get_connection(
|
|
1637
|
-
|
|
1691
|
+
conceptual_property,
|
|
1692
|
+
value_type,
|
|
1693
|
+
edge_value_types_by_concept_property_pair,
|
|
1694
|
+
default_space,
|
|
1695
|
+
default_version,
|
|
1638
1696
|
)
|
|
1639
1697
|
|
|
1640
1698
|
container: ContainerEntity | None = None
|
|
1641
1699
|
container_property: str | None = None
|
|
1642
1700
|
# DMS should have min count of either 0 or 1
|
|
1643
|
-
min_count = min(1, max(0,
|
|
1644
|
-
max_count =
|
|
1701
|
+
min_count = min(1, max(0, conceptual_property.min_count or 0))
|
|
1702
|
+
max_count = conceptual_property.max_count
|
|
1645
1703
|
if isinstance(connection, EdgeEntity):
|
|
1646
1704
|
min_count = 0
|
|
1647
1705
|
max_count = 1 if max_count == 1 else float("inf")
|
|
@@ -1650,74 +1708,82 @@ class _InformationRulesConverter:
|
|
|
1650
1708
|
max_count = 1 if max_count == 1 else float("inf")
|
|
1651
1709
|
elif connection == "direct":
|
|
1652
1710
|
min_count = 0
|
|
1653
|
-
container, container_property = self._get_container(
|
|
1711
|
+
container, container_property = self._get_container(conceptual_property, default_space)
|
|
1654
1712
|
else:
|
|
1655
|
-
container, container_property = self._get_container(
|
|
1713
|
+
container, container_property = self._get_container(conceptual_property, default_space)
|
|
1656
1714
|
|
|
1657
|
-
|
|
1658
|
-
name=
|
|
1715
|
+
physical_property = PhysicalProperty(
|
|
1716
|
+
name=conceptual_property.name,
|
|
1659
1717
|
value_type=value_type,
|
|
1660
1718
|
min_count=min_count,
|
|
1661
1719
|
max_count=max_count,
|
|
1662
1720
|
connection=connection,
|
|
1663
|
-
default=
|
|
1721
|
+
default=conceptual_property.default,
|
|
1664
1722
|
container=container,
|
|
1665
1723
|
container_property=container_property,
|
|
1666
|
-
view=
|
|
1667
|
-
view_property=
|
|
1724
|
+
view=conceptual_property.concept.as_view_entity(default_space, default_version),
|
|
1725
|
+
view_property=conceptual_property.property_,
|
|
1668
1726
|
)
|
|
1669
1727
|
|
|
1670
1728
|
# linking
|
|
1671
|
-
|
|
1729
|
+
physical_property.conceptual = conceptual_property.neatId
|
|
1672
1730
|
|
|
1673
|
-
return
|
|
1731
|
+
return physical_property
|
|
1674
1732
|
|
|
1675
1733
|
@staticmethod
|
|
1676
|
-
def
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1734
|
+
def _customize_physical_property(
|
|
1735
|
+
conceptual_property: ConceptualProperty,
|
|
1736
|
+
physical_property: PhysicalProperty,
|
|
1737
|
+
concept: ConceptEntity,
|
|
1680
1738
|
default_space: str,
|
|
1681
1739
|
default_version: str,
|
|
1682
1740
|
ancestors_by_view: dict[ViewEntity, set[ViewEntity]],
|
|
1683
|
-
) ->
|
|
1684
|
-
"""Customize the
|
|
1685
|
-
This means updating the name and description of the
|
|
1686
|
-
In addition, the value type can be updated given that the value type is matches the
|
|
1741
|
+
) -> PhysicalProperty:
|
|
1742
|
+
"""Customize the physical property to match the conceptual property.
|
|
1743
|
+
This means updating the name and description of the physical property with the conceptual property.
|
|
1744
|
+
In addition, the value type can be updated given that the value type is matches the physical property value
|
|
1687
1745
|
type or in the case of a View Value type a derivative of the cognite property value type.
|
|
1688
1746
|
|
|
1689
1747
|
Args:
|
|
1690
1748
|
prop: Information property
|
|
1691
1749
|
cognite_prop: Cognite property
|
|
1692
|
-
|
|
1750
|
+
concept: Concept entity
|
|
1693
1751
|
default_space: The default space
|
|
1694
1752
|
default_version: The default version
|
|
1695
1753
|
ancestors_by_view: Ancestors by view
|
|
1696
1754
|
|
|
1697
1755
|
Returns:
|
|
1698
|
-
|
|
1756
|
+
PhysicalProperty: The customized physical property
|
|
1699
1757
|
|
|
1700
1758
|
"""
|
|
1701
|
-
value_type: DataType | ViewEntity |
|
|
1702
|
-
if isinstance(
|
|
1759
|
+
value_type: DataType | ViewEntity | PhysicalUnknownEntity = physical_property.value_type
|
|
1760
|
+
if isinstance(conceptual_property.value_type, DataType) and conceptual_property.value_type != value_type:
|
|
1703
1761
|
warnings.warn(
|
|
1704
|
-
PropertyOverwritingWarning(
|
|
1762
|
+
PropertyOverwritingWarning(
|
|
1763
|
+
conceptual_property.property_,
|
|
1764
|
+
"property",
|
|
1765
|
+
"value type",
|
|
1766
|
+
(str(conceptual_property.value_type),),
|
|
1767
|
+
),
|
|
1705
1768
|
stacklevel=2,
|
|
1706
1769
|
)
|
|
1707
|
-
elif isinstance(
|
|
1770
|
+
elif isinstance(conceptual_property.value_type, DataType):
|
|
1708
1771
|
# User set the same value type as core concept.
|
|
1709
1772
|
pass
|
|
1710
|
-
elif isinstance(
|
|
1711
|
-
|
|
1773
|
+
elif isinstance(conceptual_property.value_type, ConceptEntity) and isinstance(
|
|
1774
|
+
physical_property.value_type, ViewEntity
|
|
1775
|
+
):
|
|
1776
|
+
view_type = conceptual_property.value_type.as_view_entity(default_space, default_version)
|
|
1712
1777
|
ancestors = ancestors_by_view.get(view_type, set())
|
|
1713
|
-
if view_type ==
|
|
1778
|
+
if view_type == physical_property.value_type or physical_property.value_type in ancestors:
|
|
1714
1779
|
value_type = view_type
|
|
1715
1780
|
else:
|
|
1716
1781
|
warnings.warn(
|
|
1717
1782
|
NeatValueWarning(
|
|
1718
1783
|
f"Invalid Value Type. The view {view_type} must implement "
|
|
1719
1784
|
f"{humanize_collection(ancestors, bind_word='or')} "
|
|
1720
|
-
|
|
1785
|
+
"to be used as the Value Type in the "
|
|
1786
|
+
f"{conceptual_property.concept!s}.{conceptual_property.property_}. "
|
|
1721
1787
|
f"Skipping..."
|
|
1722
1788
|
),
|
|
1723
1789
|
stacklevel=2,
|
|
@@ -1725,101 +1791,112 @@ class _InformationRulesConverter:
|
|
|
1725
1791
|
else:
|
|
1726
1792
|
warnings.warn(
|
|
1727
1793
|
NeatValueWarning(
|
|
1728
|
-
f"Invalid Value Type. The {
|
|
1729
|
-
f"
|
|
1794
|
+
f"Invalid Value Type. The {conceptual_property.value_type} is "
|
|
1795
|
+
f"not supported as {conceptual_property.concept} implements"
|
|
1796
|
+
"a cognite concepts. Will skip this, and use "
|
|
1797
|
+
f"the {physical_property.value_type} instead."
|
|
1730
1798
|
),
|
|
1731
1799
|
stacklevel=2,
|
|
1732
1800
|
)
|
|
1733
1801
|
|
|
1734
|
-
return
|
|
1802
|
+
return physical_property.model_copy(
|
|
1735
1803
|
update={
|
|
1736
|
-
"view":
|
|
1737
|
-
"name":
|
|
1738
|
-
"description":
|
|
1804
|
+
"view": concept.as_view_entity(default_space, default_version),
|
|
1805
|
+
"name": conceptual_property.name or physical_property.name,
|
|
1806
|
+
"description": conceptual_property.description or physical_property.description,
|
|
1739
1807
|
"value_type": value_type,
|
|
1740
1808
|
}
|
|
1741
1809
|
)
|
|
1742
1810
|
|
|
1743
1811
|
@staticmethod
|
|
1744
1812
|
def _get_connection(
|
|
1745
|
-
|
|
1746
|
-
value_type: DataType | ViewEntity |
|
|
1747
|
-
|
|
1813
|
+
conceptual_property: ConceptualProperty,
|
|
1814
|
+
value_type: DataType | ViewEntity | PhysicalUnknownEntity,
|
|
1815
|
+
edge_value_types_by_concept_property_pair: dict[tuple[ConceptEntity, str], ConceptEntity],
|
|
1748
1816
|
default_space: str,
|
|
1749
1817
|
default_version: str,
|
|
1750
1818
|
) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
|
|
1751
1819
|
if (
|
|
1752
1820
|
isinstance(value_type, ViewEntity)
|
|
1753
|
-
and (
|
|
1821
|
+
and (conceptual_property.concept, conceptual_property.property_)
|
|
1822
|
+
in edge_value_types_by_concept_property_pair
|
|
1754
1823
|
):
|
|
1755
|
-
edge_value_type =
|
|
1824
|
+
edge_value_type = edge_value_types_by_concept_property_pair[
|
|
1825
|
+
(conceptual_property.concept, conceptual_property.property_)
|
|
1826
|
+
]
|
|
1756
1827
|
return EdgeEntity(properties=edge_value_type.as_view_entity(default_space, default_version))
|
|
1757
1828
|
if isinstance(value_type, ViewEntity) and (
|
|
1758
|
-
|
|
1829
|
+
conceptual_property.max_count in {float("inf"), None}
|
|
1830
|
+
or (isinstance(conceptual_property.max_count, int | float) and conceptual_property.max_count > 1)
|
|
1759
1831
|
):
|
|
1760
1832
|
return EdgeEntity()
|
|
1761
1833
|
elif isinstance(value_type, ViewEntity):
|
|
1762
1834
|
return "direct"
|
|
1763
1835
|
# defaulting to direct connection
|
|
1764
|
-
elif isinstance(value_type,
|
|
1836
|
+
elif isinstance(value_type, PhysicalUnknownEntity):
|
|
1765
1837
|
return "direct"
|
|
1766
1838
|
return None
|
|
1767
1839
|
|
|
1768
1840
|
def _get_value_type(
|
|
1769
1841
|
self,
|
|
1770
|
-
|
|
1842
|
+
conceptual_property: ConceptualProperty,
|
|
1771
1843
|
default_space: str,
|
|
1772
1844
|
default_version: str,
|
|
1773
|
-
edge_classes: set[
|
|
1774
|
-
end_node_by_edge: dict[
|
|
1775
|
-
) -> DataType | ViewEntity |
|
|
1776
|
-
if isinstance(
|
|
1777
|
-
return
|
|
1845
|
+
edge_classes: set[ConceptEntity],
|
|
1846
|
+
end_node_by_edge: dict[ConceptEntity, ConceptEntity],
|
|
1847
|
+
) -> DataType | ViewEntity | PhysicalUnknownEntity:
|
|
1848
|
+
if isinstance(conceptual_property.value_type, DataType):
|
|
1849
|
+
return conceptual_property.value_type
|
|
1778
1850
|
|
|
1779
1851
|
# UnknownEntity should resolve to DMSUnknownEntity
|
|
1780
1852
|
# meaning end node type is unknown
|
|
1781
|
-
elif isinstance(
|
|
1782
|
-
return
|
|
1853
|
+
elif isinstance(conceptual_property.value_type, UnknownEntity):
|
|
1854
|
+
return PhysicalUnknownEntity()
|
|
1783
1855
|
|
|
1784
|
-
elif isinstance(
|
|
1785
|
-
|
|
1786
|
-
|
|
1856
|
+
elif isinstance(conceptual_property.value_type, ConceptEntity) and (
|
|
1857
|
+
conceptual_property.value_type in edge_classes
|
|
1858
|
+
):
|
|
1859
|
+
if conceptual_property.value_type in end_node_by_edge:
|
|
1860
|
+
return end_node_by_edge[conceptual_property.value_type].as_view_entity(default_space, default_version)
|
|
1787
1861
|
# This occurs if the end node is not pointing to a class
|
|
1788
1862
|
warnings.warn(
|
|
1789
1863
|
NeatValueWarning(
|
|
1790
|
-
f"Edge class {
|
|
1864
|
+
f"Edge class {conceptual_property.value_type} does not "
|
|
1865
|
+
"have 'endNode' property, defaulting to DMSUnknownEntity"
|
|
1791
1866
|
),
|
|
1792
1867
|
stacklevel=2,
|
|
1793
1868
|
)
|
|
1794
|
-
return
|
|
1795
|
-
elif isinstance(
|
|
1796
|
-
return
|
|
1869
|
+
return PhysicalUnknownEntity()
|
|
1870
|
+
elif isinstance(conceptual_property.value_type, ConceptEntity):
|
|
1871
|
+
return conceptual_property.value_type.as_view_entity(default_space, default_version)
|
|
1797
1872
|
|
|
1798
|
-
elif isinstance(
|
|
1873
|
+
elif isinstance(conceptual_property.value_type, MultiValueTypeInfo):
|
|
1799
1874
|
# Multi Object type should resolve to DMSUnknownEntity
|
|
1800
1875
|
# meaning end node type is unknown
|
|
1801
|
-
if
|
|
1802
|
-
non_unknown = [
|
|
1876
|
+
if conceptual_property.value_type.is_multi_object_type():
|
|
1877
|
+
non_unknown = [
|
|
1878
|
+
type_ for type_ in conceptual_property.value_type.types if isinstance(type_, UnknownEntity)
|
|
1879
|
+
]
|
|
1803
1880
|
if list(non_unknown) == 1:
|
|
1804
1881
|
#
|
|
1805
1882
|
return non_unknown[0].as_view_entity(default_space, default_version)
|
|
1806
|
-
return
|
|
1883
|
+
return PhysicalUnknownEntity()
|
|
1807
1884
|
|
|
1808
1885
|
# Multi Data type should resolve to a single data type, or it should
|
|
1809
|
-
elif
|
|
1810
|
-
return self.convert_multi_data_type(
|
|
1886
|
+
elif conceptual_property.value_type.is_multi_data_type():
|
|
1887
|
+
return self.convert_multi_data_type(conceptual_property.value_type)
|
|
1811
1888
|
|
|
1812
1889
|
# Mixed types default to string
|
|
1813
1890
|
else:
|
|
1814
|
-
non_any_uri = [type_ for type_ in
|
|
1891
|
+
non_any_uri = [type_ for type_ in conceptual_property.value_type.types if type_ != AnyURI()]
|
|
1815
1892
|
if list(non_any_uri) == 1:
|
|
1816
|
-
if isinstance(non_any_uri[0],
|
|
1893
|
+
if isinstance(non_any_uri[0], ConceptEntity):
|
|
1817
1894
|
return non_any_uri[0].as_view_entity(default_space, default_version)
|
|
1818
1895
|
else:
|
|
1819
1896
|
return non_any_uri[0]
|
|
1820
1897
|
return String()
|
|
1821
1898
|
|
|
1822
|
-
raise ValueError(f"Unsupported value type: {
|
|
1899
|
+
raise ValueError(f"Unsupported value type: {conceptual_property.value_type.type_}")
|
|
1823
1900
|
|
|
1824
1901
|
@classmethod
|
|
1825
1902
|
def _to_space(cls, prefix: str) -> str:
|
|
@@ -1832,8 +1909,8 @@ class _InformationRulesConverter:
|
|
|
1832
1909
|
prefix = f"{prefix[:-1]}1"
|
|
1833
1910
|
return prefix
|
|
1834
1911
|
|
|
1835
|
-
def _get_container(self, prop:
|
|
1836
|
-
container_entity = prop.
|
|
1912
|
+
def _get_container(self, prop: ConceptualProperty, default_space: str) -> tuple[ContainerEntity, str]:
|
|
1913
|
+
container_entity = prop.concept.as_container_entity(default_space)
|
|
1837
1914
|
|
|
1838
1915
|
while self.property_count_by_container[container_entity] >= DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
1839
1916
|
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
@@ -1841,7 +1918,7 @@ class _InformationRulesConverter:
|
|
|
1841
1918
|
self.property_count_by_container[container_entity] += 1
|
|
1842
1919
|
return container_entity, prop.property_
|
|
1843
1920
|
|
|
1844
|
-
def _get_view_implements(self, cls_:
|
|
1921
|
+
def _get_view_implements(self, cls_: Concept, metadata: ConceptualMetadata) -> list[ViewEntity]:
|
|
1845
1922
|
implements = []
|
|
1846
1923
|
for parent in cls_.implements or []:
|
|
1847
1924
|
view_entity = parent.as_view_entity(metadata.prefix, metadata.version)
|
|
@@ -1880,16 +1957,18 @@ class _InformationRulesConverter:
|
|
|
1880
1957
|
|
|
1881
1958
|
return data_types.String()
|
|
1882
1959
|
|
|
1883
|
-
def _get_cognite_concepts(self) -> set[
|
|
1884
|
-
return {
|
|
1960
|
+
def _get_cognite_concepts(self) -> set[ConceptEntity]:
|
|
1961
|
+
return {
|
|
1962
|
+
cls_.concept for cls_ in self.conceptual_data_model.concepts if str(cls_.concept.prefix) in COGNITE_SPACES
|
|
1963
|
+
} | {
|
|
1885
1964
|
parent
|
|
1886
|
-
for cls_ in self.
|
|
1965
|
+
for cls_ in self.conceptual_data_model.concepts
|
|
1887
1966
|
for parent in cls_.implements or []
|
|
1888
1967
|
if str(parent.prefix) in COGNITE_SPACES
|
|
1889
1968
|
}
|
|
1890
1969
|
|
|
1891
1970
|
@staticmethod
|
|
1892
|
-
def
|
|
1971
|
+
def _get_cognite_physical_data_model(concepts: set[ConceptEntity], client: NeatClient) -> PhysicalDataModel:
|
|
1893
1972
|
view_ids = [dm.ViewId(str(cls_.prefix), cls_.suffix, cls_.version) for cls_ in concepts]
|
|
1894
1973
|
views = client.loaders.views.retrieve(view_ids, format="read", include_connected=True, include_ancestor=True)
|
|
1895
1974
|
spaces = Counter(view.space for view in views)
|
|
@@ -1907,15 +1986,17 @@ class _InformationRulesConverter:
|
|
|
1907
1986
|
"purpose.",
|
|
1908
1987
|
views=list(views),
|
|
1909
1988
|
)
|
|
1910
|
-
unverified = DMSImporter.from_data_model(client, model).
|
|
1911
|
-
if unverified.
|
|
1989
|
+
unverified = DMSImporter.from_data_model(client, model).to_data_model()
|
|
1990
|
+
if unverified.unverified_data_model is None:
|
|
1912
1991
|
raise NeatValueError("Failed to create CogniteConcepts")
|
|
1913
|
-
return unverified.
|
|
1992
|
+
return unverified.unverified_data_model.as_verified_data_model()
|
|
1914
1993
|
|
|
1915
1994
|
@staticmethod
|
|
1916
1995
|
def _find_cognite_property(
|
|
1917
|
-
property_: str,
|
|
1918
|
-
|
|
1996
|
+
property_: str,
|
|
1997
|
+
parents: set[ConceptEntity],
|
|
1998
|
+
cognite_properties: dict[tuple[ConceptEntity, str], PhysicalProperty],
|
|
1999
|
+
) -> PhysicalProperty | None:
|
|
1919
2000
|
"""Find the parent class that has the property in the cognite properties"""
|
|
1920
2001
|
for parent in parents:
|
|
1921
2002
|
if (parent, property_) in cognite_properties:
|
|
@@ -1924,40 +2005,38 @@ class _InformationRulesConverter:
|
|
|
1924
2005
|
|
|
1925
2006
|
|
|
1926
2007
|
class _DMSRulesConverter:
|
|
1927
|
-
def __init__(self,
|
|
1928
|
-
self.
|
|
2008
|
+
def __init__(self, data_model: PhysicalDataModel, instance_namespace: Namespace | None = None) -> None:
|
|
2009
|
+
self.physical_data_model = data_model
|
|
1929
2010
|
self.instance_namespace = instance_namespace
|
|
1930
2011
|
|
|
1931
|
-
def
|
|
2012
|
+
def as_conceptual_data_model(
|
|
1932
2013
|
self,
|
|
1933
|
-
) -> "
|
|
1934
|
-
from cognite.neat.core.
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
2014
|
+
) -> "ConceptualDataModel":
|
|
2015
|
+
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
2016
|
+
Concept,
|
|
2017
|
+
ConceptualDataModel,
|
|
2018
|
+
ConceptualProperty,
|
|
1938
2019
|
)
|
|
1939
2020
|
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
metadata = self._convert_metadata_to_info(dms)
|
|
2021
|
+
metadata = self._convert_physical_to_conceptual_metadata(self.physical_data_model.metadata)
|
|
1943
2022
|
|
|
1944
|
-
|
|
1945
|
-
for view in self.
|
|
1946
|
-
|
|
2023
|
+
concepts: list[Concept] = []
|
|
2024
|
+
for view in self.physical_data_model.views:
|
|
2025
|
+
concept = Concept(
|
|
1947
2026
|
# we do not want a version in class as we use URI for the class
|
|
1948
|
-
|
|
2027
|
+
concept=ConceptEntity(prefix=view.view.prefix, suffix=view.view.suffix),
|
|
1949
2028
|
description=view.description,
|
|
1950
2029
|
name=view.name,
|
|
1951
2030
|
implements=[
|
|
1952
2031
|
# we do not want a version in class as we use URI for the class
|
|
1953
|
-
implemented_view.
|
|
2032
|
+
implemented_view.as_concept_entity(skip_version=True)
|
|
1954
2033
|
for implemented_view in view.implements or []
|
|
1955
2034
|
],
|
|
1956
2035
|
)
|
|
1957
2036
|
|
|
1958
2037
|
# Linking
|
|
1959
|
-
|
|
1960
|
-
|
|
2038
|
+
concept.physical = view.neatId
|
|
2039
|
+
concepts.append(concept)
|
|
1961
2040
|
|
|
1962
2041
|
prefixes = get_default_prefixes_and_namespaces()
|
|
1963
2042
|
if self.instance_namespace:
|
|
@@ -1967,24 +2046,24 @@ class _DMSRulesConverter:
|
|
|
1967
2046
|
instance_prefix = f"prefix_{len(prefixes) + 1}"
|
|
1968
2047
|
prefixes[instance_prefix] = self.instance_namespace
|
|
1969
2048
|
|
|
1970
|
-
properties: list[
|
|
1971
|
-
value_type: DataType |
|
|
1972
|
-
for property_ in self.
|
|
2049
|
+
properties: list[ConceptualProperty] = []
|
|
2050
|
+
value_type: DataType | ConceptEntity | str
|
|
2051
|
+
for property_ in self.physical_data_model.properties:
|
|
1973
2052
|
if isinstance(property_.value_type, DataType):
|
|
1974
2053
|
value_type = property_.value_type
|
|
1975
2054
|
elif isinstance(property_.value_type, ViewEntity):
|
|
1976
|
-
value_type =
|
|
2055
|
+
value_type = ConceptEntity(
|
|
1977
2056
|
prefix=property_.value_type.prefix,
|
|
1978
2057
|
suffix=property_.value_type.suffix,
|
|
1979
2058
|
)
|
|
1980
|
-
elif isinstance(property_.value_type,
|
|
2059
|
+
elif isinstance(property_.value_type, PhysicalUnknownEntity):
|
|
1981
2060
|
value_type = UnknownEntity()
|
|
1982
2061
|
else:
|
|
1983
2062
|
raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
|
|
1984
2063
|
|
|
1985
|
-
|
|
2064
|
+
conceptual_property = ConceptualProperty(
|
|
1986
2065
|
# Removing version
|
|
1987
|
-
|
|
2066
|
+
concept=ConceptEntity(suffix=property_.view.suffix, prefix=property_.view.prefix),
|
|
1988
2067
|
property_=property_.view_property,
|
|
1989
2068
|
name=property_.name,
|
|
1990
2069
|
value_type=value_type,
|
|
@@ -1994,28 +2073,28 @@ class _DMSRulesConverter:
|
|
|
1994
2073
|
)
|
|
1995
2074
|
|
|
1996
2075
|
# Linking
|
|
1997
|
-
|
|
2076
|
+
conceptual_property.physical = property_.neatId
|
|
1998
2077
|
|
|
1999
|
-
properties.append(
|
|
2078
|
+
properties.append(conceptual_property)
|
|
2000
2079
|
|
|
2001
|
-
|
|
2080
|
+
conceptual_data_model = ConceptualDataModel(
|
|
2002
2081
|
metadata=metadata,
|
|
2003
|
-
properties=SheetList[
|
|
2004
|
-
|
|
2082
|
+
properties=SheetList[ConceptualProperty](properties),
|
|
2083
|
+
concepts=SheetList[Concept](concepts),
|
|
2005
2084
|
prefixes=prefixes,
|
|
2006
2085
|
)
|
|
2007
2086
|
|
|
2008
|
-
self.
|
|
2087
|
+
self.physical_data_model.sync_with_conceptual_data_model(conceptual_data_model)
|
|
2009
2088
|
|
|
2010
|
-
return
|
|
2089
|
+
return conceptual_data_model
|
|
2011
2090
|
|
|
2012
2091
|
@classmethod
|
|
2013
|
-
def
|
|
2014
|
-
from cognite.neat.core.
|
|
2015
|
-
|
|
2092
|
+
def _convert_physical_to_conceptual_metadata(cls, metadata: PhysicalMetadata) -> "ConceptualMetadata":
|
|
2093
|
+
from cognite.neat.core._data_model.models.conceptual._verified import (
|
|
2094
|
+
ConceptualMetadata,
|
|
2016
2095
|
)
|
|
2017
2096
|
|
|
2018
|
-
return
|
|
2097
|
+
return ConceptualMetadata(
|
|
2019
2098
|
space=metadata.space,
|
|
2020
2099
|
external_id=metadata.external_id,
|
|
2021
2100
|
version=metadata.version,
|
|
@@ -2027,8 +2106,8 @@ class _DMSRulesConverter:
|
|
|
2027
2106
|
)
|
|
2028
2107
|
|
|
2029
2108
|
|
|
2030
|
-
class _SubsetEditableCDMRules(
|
|
2031
|
-
"""Subsets editable CDM
|
|
2109
|
+
class _SubsetEditableCDMRules(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
2110
|
+
"""Subsets editable CDM data model to only include desired set of CDM concepts.
|
|
2032
2111
|
|
|
2033
2112
|
!!! note "Platypus UI limitations"
|
|
2034
2113
|
This is temporal solution to enable cleaner extension of core data model,
|
|
@@ -2044,65 +2123,65 @@ class _SubsetEditableCDMRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
2044
2123
|
|
|
2045
2124
|
self._views = views
|
|
2046
2125
|
|
|
2047
|
-
def transform(self,
|
|
2126
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
2048
2127
|
# should check to make sure data model is based on the editable CDM
|
|
2049
2128
|
# if not raise an error
|
|
2050
2129
|
|
|
2051
|
-
|
|
2052
|
-
"metadata":
|
|
2053
|
-
"views": SheetList[
|
|
2054
|
-
"properties": SheetList[
|
|
2055
|
-
"containers": SheetList[
|
|
2056
|
-
"enum":
|
|
2057
|
-
"nodes":
|
|
2130
|
+
subsetted_data_model: dict[str, Any] = {
|
|
2131
|
+
"metadata": data_model.metadata.model_copy(),
|
|
2132
|
+
"views": SheetList[PhysicalView](),
|
|
2133
|
+
"properties": SheetList[PhysicalProperty](),
|
|
2134
|
+
"containers": SheetList[PhysicalContainer](),
|
|
2135
|
+
"enum": data_model.enum,
|
|
2136
|
+
"nodes": data_model.nodes,
|
|
2058
2137
|
}
|
|
2059
2138
|
|
|
2060
2139
|
containers_to_keep = set()
|
|
2061
2140
|
|
|
2062
|
-
if editable_views_to_keep := self._editable_views_to_keep(
|
|
2063
|
-
for view in
|
|
2141
|
+
if editable_views_to_keep := self._editable_views_to_keep(data_model):
|
|
2142
|
+
for view in data_model.views:
|
|
2064
2143
|
if view.view in editable_views_to_keep or view.view.space in COGNITE_SPACES:
|
|
2065
|
-
|
|
2144
|
+
subsetted_data_model["views"].append(view)
|
|
2066
2145
|
|
|
2067
|
-
for property_ in
|
|
2146
|
+
for property_ in data_model.properties:
|
|
2068
2147
|
if property_.view in editable_views_to_keep and (
|
|
2069
2148
|
isinstance(property_.value_type, DataType)
|
|
2070
|
-
or isinstance(property_.value_type,
|
|
2149
|
+
or isinstance(property_.value_type, PhysicalUnknownEntity)
|
|
2071
2150
|
or (isinstance(property_.value_type, ViewEntity) and property_.value_type in editable_views_to_keep)
|
|
2072
2151
|
):
|
|
2073
|
-
|
|
2152
|
+
subsetted_data_model["properties"].append(property_)
|
|
2074
2153
|
if property_.container:
|
|
2075
2154
|
containers_to_keep.add(property_.container)
|
|
2076
2155
|
|
|
2077
|
-
if
|
|
2078
|
-
for container in
|
|
2156
|
+
if data_model.containers:
|
|
2157
|
+
for container in data_model.containers:
|
|
2079
2158
|
if container.container in containers_to_keep:
|
|
2080
|
-
|
|
2159
|
+
subsetted_data_model["containers"].append(container)
|
|
2081
2160
|
try:
|
|
2082
|
-
return
|
|
2161
|
+
return PhysicalDataModel.model_validate(subsetted_data_model)
|
|
2083
2162
|
except ValidationError as e:
|
|
2084
|
-
raise NeatValueError(f"Cannot subset
|
|
2163
|
+
raise NeatValueError(f"Cannot subset data_model: {e}") from e
|
|
2085
2164
|
else:
|
|
2086
|
-
raise NeatValueError("Cannot subset
|
|
2165
|
+
raise NeatValueError("Cannot subset data_model: provided data model is not based on Core Data Model")
|
|
2087
2166
|
|
|
2088
|
-
def _editable_views_to_keep(self,
|
|
2167
|
+
def _editable_views_to_keep(self, data_model: PhysicalDataModel) -> set[ViewEntity]:
|
|
2089
2168
|
return {
|
|
2090
2169
|
view.view
|
|
2091
|
-
for view in
|
|
2170
|
+
for view in data_model.views
|
|
2092
2171
|
if view.view.space not in COGNITE_SPACES
|
|
2093
2172
|
and view.implements
|
|
2094
2173
|
and any(implemented in self._views for implemented in view.implements)
|
|
2095
2174
|
}
|
|
2096
2175
|
|
|
2097
2176
|
|
|
2098
|
-
class
|
|
2099
|
-
"""Subsets
|
|
2177
|
+
class SubsetPhysicalDataModel(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
|
|
2178
|
+
"""Subsets physical data model to only include the specified views."""
|
|
2100
2179
|
|
|
2101
2180
|
def __init__(self, views: set[ViewEntity]):
|
|
2102
2181
|
self._views = views
|
|
2103
2182
|
|
|
2104
|
-
def transform(self,
|
|
2105
|
-
analysis =
|
|
2183
|
+
def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
|
|
2184
|
+
analysis = DataModelAnalysis(physical=data_model)
|
|
2106
2185
|
|
|
2107
2186
|
views_by_view = analysis.view_by_view_entity
|
|
2108
2187
|
implements_by_view = analysis.implements_by_view()
|
|
@@ -2116,26 +2195,26 @@ class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
2116
2195
|
subset = subset.union(ancestors)
|
|
2117
2196
|
|
|
2118
2197
|
if not subset:
|
|
2119
|
-
raise NeatValueError("None of the requested views are defined in the
|
|
2198
|
+
raise NeatValueError("None of the requested views are defined in the data_model!")
|
|
2120
2199
|
|
|
2121
2200
|
if nonexisting := self._views - subset:
|
|
2122
2201
|
raise NeatValueError(
|
|
2123
2202
|
"Following requested views do not exist"
|
|
2124
|
-
f" in the
|
|
2203
|
+
f" in the data_model: [{','.join([view.external_id for view in nonexisting])}]. Aborting."
|
|
2125
2204
|
)
|
|
2126
2205
|
|
|
2127
|
-
|
|
2128
|
-
"metadata":
|
|
2129
|
-
"views": SheetList[
|
|
2130
|
-
"properties": SheetList[
|
|
2131
|
-
"containers": SheetList[
|
|
2132
|
-
"enum":
|
|
2133
|
-
"nodes":
|
|
2206
|
+
subsetted_data_model: dict[str, Any] = {
|
|
2207
|
+
"metadata": data_model.metadata.model_copy(),
|
|
2208
|
+
"views": SheetList[PhysicalView](),
|
|
2209
|
+
"properties": SheetList[PhysicalProperty](),
|
|
2210
|
+
"containers": SheetList[PhysicalContainer](),
|
|
2211
|
+
"enum": data_model.enum,
|
|
2212
|
+
"nodes": data_model.nodes,
|
|
2134
2213
|
}
|
|
2135
2214
|
|
|
2136
2215
|
# add views
|
|
2137
2216
|
for view in subset:
|
|
2138
|
-
|
|
2217
|
+
subsetted_data_model["views"].append(views_by_view[view])
|
|
2139
2218
|
|
|
2140
2219
|
used_containers = set()
|
|
2141
2220
|
|
|
@@ -2147,85 +2226,85 @@ class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
|
|
|
2147
2226
|
for property_ in properties:
|
|
2148
2227
|
if (
|
|
2149
2228
|
isinstance(property_.value_type, DataType)
|
|
2150
|
-
or isinstance(property_.value_type,
|
|
2229
|
+
or isinstance(property_.value_type, PhysicalUnknownEntity)
|
|
2151
2230
|
or (isinstance(property_.value_type, ViewEntity) and property_.value_type in subset)
|
|
2152
2231
|
):
|
|
2153
|
-
|
|
2232
|
+
subsetted_data_model["properties"].append(property_)
|
|
2154
2233
|
|
|
2155
2234
|
if property_.container:
|
|
2156
2235
|
used_containers.add(property_.container)
|
|
2157
2236
|
|
|
2158
2237
|
# add containers
|
|
2159
|
-
if
|
|
2160
|
-
for container in
|
|
2238
|
+
if data_model.containers:
|
|
2239
|
+
for container in data_model.containers:
|
|
2161
2240
|
if container.container in used_containers:
|
|
2162
|
-
|
|
2241
|
+
subsetted_data_model["containers"].append(container)
|
|
2163
2242
|
|
|
2164
2243
|
try:
|
|
2165
|
-
return
|
|
2244
|
+
return PhysicalDataModel.model_validate(subsetted_data_model)
|
|
2166
2245
|
except ValidationError as e:
|
|
2167
|
-
raise NeatValueError(f"Cannot subset
|
|
2246
|
+
raise NeatValueError(f"Cannot subset data_model: {e}") from e
|
|
2168
2247
|
|
|
2169
2248
|
|
|
2170
|
-
class
|
|
2171
|
-
"""Subsets
|
|
2249
|
+
class SubsetConceptualDataModel(VerifiedDataModelTransformer[ConceptualDataModel, ConceptualDataModel]):
|
|
2250
|
+
"""Subsets Conceptual Data Model to only include the specified concepts."""
|
|
2172
2251
|
|
|
2173
|
-
def __init__(self,
|
|
2174
|
-
self.
|
|
2252
|
+
def __init__(self, concepts: set[ConceptEntity]):
|
|
2253
|
+
self._concepts = concepts
|
|
2175
2254
|
|
|
2176
|
-
def transform(self,
|
|
2177
|
-
analysis =
|
|
2255
|
+
def transform(self, data_model: ConceptualDataModel) -> ConceptualDataModel:
|
|
2256
|
+
analysis = DataModelAnalysis(conceptual=data_model)
|
|
2178
2257
|
|
|
2179
|
-
|
|
2180
|
-
|
|
2258
|
+
concept_by_concept_entity = analysis.concept_by_concept_entity
|
|
2259
|
+
parent_entity_by_concept_entity = analysis.parents_by_concept()
|
|
2181
2260
|
|
|
2182
|
-
available = analysis.
|
|
2183
|
-
subset = available.intersection(self.
|
|
2261
|
+
available = analysis.defined_concepts(include_ancestors=True)
|
|
2262
|
+
subset = available.intersection(self._concepts)
|
|
2184
2263
|
|
|
2185
2264
|
# need to add all the parent classes of the desired classes to the possible classes
|
|
2186
|
-
ancestors: set[
|
|
2187
|
-
for
|
|
2265
|
+
ancestors: set[ConceptEntity] = set()
|
|
2266
|
+
for concept in subset:
|
|
2188
2267
|
ancestors = ancestors.union(
|
|
2189
|
-
{ancestor for ancestor in get_inheritance_path(
|
|
2268
|
+
{ancestor for ancestor in get_inheritance_path(concept, parent_entity_by_concept_entity)}
|
|
2190
2269
|
)
|
|
2191
2270
|
subset = subset.union(ancestors)
|
|
2192
2271
|
|
|
2193
2272
|
if not subset:
|
|
2194
|
-
raise NeatValueError("None of the requested
|
|
2273
|
+
raise NeatValueError("None of the requested concepts are defined in the data_model!")
|
|
2195
2274
|
|
|
2196
|
-
if nonexisting := self.
|
|
2275
|
+
if nonexisting := self._concepts - subset:
|
|
2197
2276
|
raise NeatValueError(
|
|
2198
|
-
"Following requested
|
|
2199
|
-
f" in the
|
|
2277
|
+
"Following requested concepts do not exist"
|
|
2278
|
+
f" in the data_model: [{','.join([concept.suffix for concept in nonexisting])}]"
|
|
2200
2279
|
". Aborting."
|
|
2201
2280
|
)
|
|
2202
2281
|
|
|
2203
|
-
|
|
2204
|
-
"metadata":
|
|
2205
|
-
"prefixes": (
|
|
2206
|
-
"
|
|
2207
|
-
"properties": SheetList[
|
|
2282
|
+
subsetted_data_model: dict[str, Any] = {
|
|
2283
|
+
"metadata": data_model.metadata.model_copy(),
|
|
2284
|
+
"prefixes": (data_model.prefixes or {}).copy(),
|
|
2285
|
+
"concepts": SheetList[Concept](),
|
|
2286
|
+
"properties": SheetList[ConceptualProperty](),
|
|
2208
2287
|
}
|
|
2209
2288
|
|
|
2210
|
-
for
|
|
2211
|
-
|
|
2289
|
+
for concept in subset:
|
|
2290
|
+
subsetted_data_model["concepts"].append(concept_by_concept_entity[concept])
|
|
2212
2291
|
|
|
2213
|
-
for
|
|
2214
|
-
if
|
|
2292
|
+
for concept, properties in analysis.properties_by_concepts(include_ancestors=False).items():
|
|
2293
|
+
if concept not in subset:
|
|
2215
2294
|
continue
|
|
2216
2295
|
for property_ in properties:
|
|
2217
2296
|
# datatype property can be added directly
|
|
2218
2297
|
if (
|
|
2219
2298
|
isinstance(property_.value_type, DataType)
|
|
2220
|
-
or (isinstance(property_.value_type,
|
|
2299
|
+
or (isinstance(property_.value_type, ConceptEntity) and property_.value_type in subset)
|
|
2221
2300
|
or isinstance(property_.value_type, UnknownEntity)
|
|
2222
2301
|
):
|
|
2223
|
-
|
|
2302
|
+
subsetted_data_model["properties"].append(property_)
|
|
2224
2303
|
# object property can be added if the value type is in the subset
|
|
2225
2304
|
elif isinstance(property_.value_type, MultiValueTypeInfo):
|
|
2226
2305
|
allowed = [t for t in property_.value_type.types if t in subset or isinstance(t, DataType)]
|
|
2227
2306
|
if allowed:
|
|
2228
|
-
|
|
2307
|
+
subsetted_data_model["properties"].append(
|
|
2229
2308
|
property_.model_copy(
|
|
2230
2309
|
deep=True,
|
|
2231
2310
|
update={"value_type": MultiValueTypeInfo(types=allowed)},
|
|
@@ -2233,12 +2312,17 @@ class SubsetInformationRules(VerifiedRulesTransformer[InformationRules, Informat
|
|
|
2233
2312
|
)
|
|
2234
2313
|
|
|
2235
2314
|
try:
|
|
2236
|
-
return
|
|
2315
|
+
return ConceptualDataModel.model_validate(subsetted_data_model)
|
|
2237
2316
|
except ValidationError as e:
|
|
2238
|
-
raise NeatValueError(f"Cannot subset
|
|
2317
|
+
raise NeatValueError(f"Cannot subset data_model: {e}") from e
|
|
2239
2318
|
|
|
2240
2319
|
|
|
2241
|
-
class AddCogniteProperties(
|
|
2320
|
+
class AddCogniteProperties(
|
|
2321
|
+
DataModelTransformer[
|
|
2322
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
2323
|
+
ImportedDataModel[UnverifiedConceptualDataModel],
|
|
2324
|
+
]
|
|
2325
|
+
):
|
|
2242
2326
|
"""This transformer looks at the implements of the classes and adds all properties
|
|
2243
2327
|
from the parent (and ancestors) classes that are not already included in the data model.
|
|
2244
2328
|
|
|
@@ -2257,110 +2341,125 @@ class AddCogniteProperties(RulesTransformer[ReadRules[InformationInputRules], Re
|
|
|
2257
2341
|
"""Get the description of the transformer."""
|
|
2258
2342
|
return "Add Cognite properties for all concepts that implements a Cognite concept."
|
|
2259
2343
|
|
|
2260
|
-
def transform(
|
|
2261
|
-
|
|
2344
|
+
def transform(
|
|
2345
|
+
self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]
|
|
2346
|
+
) -> ImportedDataModel[UnverifiedConceptualDataModel]:
|
|
2347
|
+
input_ = data_model.unverified_data_model
|
|
2262
2348
|
if input_ is None:
|
|
2263
|
-
raise NeatValueError("Rule read failed. Cannot add cognite properties to None
|
|
2349
|
+
raise NeatValueError("Rule read failed. Cannot add cognite properties to None data_model.")
|
|
2264
2350
|
|
|
2265
2351
|
default_space = input_.metadata.space
|
|
2266
2352
|
default_version = input_.metadata.version
|
|
2267
2353
|
|
|
2268
|
-
|
|
2269
|
-
properties_by_class = self._get_properties_by_class(input_.properties, rules.read_context, default_space)
|
|
2270
|
-
cognite_implements_concepts = self._get_cognite_concepts(dependencies_by_class)
|
|
2271
|
-
views_by_class_entity = self._get_views_by_class(cognite_implements_concepts, default_space, default_version)
|
|
2354
|
+
dependencies_by_concept = self._get_dependencies_by_concepts(input_.concepts, data_model.context, default_space)
|
|
2272
2355
|
|
|
2273
|
-
|
|
2356
|
+
properties_by_concepts = self._get_properties_by_concepts(input_.properties, data_model.context, default_space)
|
|
2357
|
+
|
|
2358
|
+
cognite_implements_concepts = self._get_cognite_concepts(dependencies_by_concept)
|
|
2359
|
+
views_by_concept_entity = self._get_views_by_concept(
|
|
2360
|
+
cognite_implements_concepts, default_space, default_version
|
|
2361
|
+
)
|
|
2362
|
+
|
|
2363
|
+
for concept_entity, view in views_by_concept_entity.items():
|
|
2274
2364
|
for prop_id, view_prop in view.properties.items():
|
|
2275
|
-
if prop_id in
|
|
2365
|
+
if prop_id in properties_by_concepts[concept_entity]:
|
|
2276
2366
|
continue
|
|
2277
|
-
|
|
2278
|
-
|
|
2367
|
+
properties_by_concepts[concept_entity][prop_id] = DMSImporter.as_unverified_conceptual_property(
|
|
2368
|
+
concept_entity, prop_id, view_prop
|
|
2279
2369
|
)
|
|
2280
2370
|
|
|
2281
2371
|
try:
|
|
2282
|
-
topological_order = TopologicalSorter(
|
|
2372
|
+
topological_order = TopologicalSorter(dependencies_by_concept).static_order()
|
|
2283
2373
|
except CycleError as e:
|
|
2284
2374
|
raise NeatValueError(f"Cycle detected in the class hierarchy: {e}") from e
|
|
2285
2375
|
|
|
2286
|
-
new_properties: list[
|
|
2287
|
-
for
|
|
2288
|
-
if
|
|
2376
|
+
new_properties: list[UnverifiedConceptualProperty] = input_.properties.copy()
|
|
2377
|
+
for concept_entity in topological_order:
|
|
2378
|
+
if concept_entity not in dependencies_by_concept:
|
|
2289
2379
|
continue
|
|
2290
|
-
for parent in
|
|
2291
|
-
for prop in
|
|
2292
|
-
if prop.property_ not in
|
|
2293
|
-
new_prop = prop.copy(
|
|
2380
|
+
for parent in dependencies_by_concept[concept_entity]:
|
|
2381
|
+
for prop in properties_by_concepts[parent].values():
|
|
2382
|
+
if prop.property_ not in properties_by_concepts[concept_entity]:
|
|
2383
|
+
new_prop = prop.copy(
|
|
2384
|
+
update={"Concept": concept_entity},
|
|
2385
|
+
default_prefix=default_space,
|
|
2386
|
+
)
|
|
2294
2387
|
new_properties.append(new_prop)
|
|
2295
|
-
|
|
2388
|
+
properties_by_concepts[concept_entity][prop.property_] = new_prop
|
|
2296
2389
|
|
|
2297
2390
|
if self._dummy_property:
|
|
2298
2391
|
new_properties.append(
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
property_=f"{to_camel_case(
|
|
2392
|
+
UnverifiedConceptualProperty(
|
|
2393
|
+
concept=concept_entity,
|
|
2394
|
+
property_=f"{to_camel_case(concept_entity.suffix)}{self._dummy_property}",
|
|
2302
2395
|
value_type=String(),
|
|
2303
2396
|
min_count=0,
|
|
2304
2397
|
max_count=1,
|
|
2305
2398
|
)
|
|
2306
2399
|
)
|
|
2307
2400
|
|
|
2308
|
-
new_classes: list[
|
|
2309
|
-
existing_classes = {cls.
|
|
2310
|
-
for
|
|
2311
|
-
if
|
|
2312
|
-
new_classes.append(DMSImporter.
|
|
2313
|
-
existing_classes.add(
|
|
2401
|
+
new_classes: list[UnverifiedConcept] = input_.concepts.copy()
|
|
2402
|
+
existing_classes = {cls.concept for cls in input_.concepts}
|
|
2403
|
+
for concept_entity, view in views_by_concept_entity.items():
|
|
2404
|
+
if concept_entity not in existing_classes:
|
|
2405
|
+
new_classes.append(DMSImporter.as_unverified_concept(view))
|
|
2406
|
+
existing_classes.add(concept_entity)
|
|
2314
2407
|
|
|
2315
|
-
return
|
|
2316
|
-
|
|
2408
|
+
return ImportedDataModel(
|
|
2409
|
+
unverified_data_model=UnverifiedConceptualDataModel(
|
|
2317
2410
|
metadata=input_.metadata,
|
|
2318
2411
|
properties=new_properties,
|
|
2319
|
-
|
|
2412
|
+
concepts=new_classes,
|
|
2320
2413
|
prefixes=input_.prefixes,
|
|
2321
2414
|
),
|
|
2322
|
-
|
|
2415
|
+
context={},
|
|
2323
2416
|
)
|
|
2324
2417
|
|
|
2325
2418
|
@staticmethod
|
|
2326
|
-
def
|
|
2327
|
-
properties: list[
|
|
2328
|
-
|
|
2419
|
+
def _get_properties_by_concepts(
|
|
2420
|
+
properties: list[UnverifiedConceptualProperty],
|
|
2421
|
+
read_context: dict[str, SpreadsheetRead],
|
|
2422
|
+
default_space: str,
|
|
2423
|
+
) -> dict[ConceptEntity, dict[str, UnverifiedConceptualProperty]]:
|
|
2329
2424
|
issues = IssueList()
|
|
2330
|
-
properties_by_class: dict[
|
|
2425
|
+
properties_by_class: dict[ConceptEntity, dict[str, UnverifiedConceptualProperty]] = defaultdict(dict)
|
|
2331
2426
|
for prop in properties:
|
|
2332
2427
|
try:
|
|
2333
2428
|
dumped = prop.dump(default_prefix=default_space)
|
|
2334
2429
|
except ValidationError as e:
|
|
2335
2430
|
issues.extend(from_pydantic_errors(e.errors(), read_context))
|
|
2336
2431
|
continue
|
|
2337
|
-
|
|
2338
|
-
properties_by_class[
|
|
2432
|
+
concept_entity = cast(ConceptEntity, dumped["Concept"])
|
|
2433
|
+
properties_by_class[concept_entity][prop.property_] = prop
|
|
2339
2434
|
if issues.has_errors:
|
|
2340
2435
|
raise issues.as_errors(operation="Reading properties")
|
|
2341
2436
|
return properties_by_class
|
|
2342
2437
|
|
|
2343
2438
|
@staticmethod
|
|
2344
|
-
def
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2439
|
+
def _get_dependencies_by_concepts(
|
|
2440
|
+
concepts: list[UnverifiedConcept],
|
|
2441
|
+
read_context: dict[str, SpreadsheetRead],
|
|
2442
|
+
default_space: str,
|
|
2443
|
+
) -> dict[ConceptEntity, set[ConceptEntity]]:
|
|
2444
|
+
dependencies_by_concepts: dict[ConceptEntity, set[ConceptEntity]] = {}
|
|
2348
2445
|
issues = IssueList()
|
|
2349
|
-
for raw in
|
|
2446
|
+
for raw in concepts:
|
|
2350
2447
|
try:
|
|
2351
2448
|
dumped = raw.dump(default_prefix=default_space)
|
|
2352
2449
|
except ValidationError as e:
|
|
2353
2450
|
issues.extend(from_pydantic_errors(e.errors(), read_context))
|
|
2354
2451
|
continue
|
|
2355
|
-
|
|
2356
|
-
implements = cast(list[
|
|
2357
|
-
|
|
2452
|
+
concept_entity = cast(ConceptEntity, dumped["Concept"])
|
|
2453
|
+
implements = cast(list[ConceptEntity] | None, dumped["Implements"])
|
|
2454
|
+
dependencies_by_concepts[concept_entity] = set(implements or [])
|
|
2358
2455
|
if issues.has_errors:
|
|
2359
2456
|
raise issues.as_errors(operation="Reading classes")
|
|
2360
|
-
return
|
|
2457
|
+
return dependencies_by_concepts
|
|
2361
2458
|
|
|
2362
2459
|
@staticmethod
|
|
2363
|
-
def _get_cognite_concepts(
|
|
2460
|
+
def _get_cognite_concepts(
|
|
2461
|
+
dependencies_by_class: dict[ConceptEntity, set[ConceptEntity]],
|
|
2462
|
+
) -> set[ConceptEntity]:
|
|
2364
2463
|
cognite_implements_concepts = {
|
|
2365
2464
|
dependency
|
|
2366
2465
|
for dependencies in dependencies_by_class.values()
|
|
@@ -2371,9 +2470,9 @@ class AddCogniteProperties(RulesTransformer[ReadRules[InformationInputRules], Re
|
|
|
2371
2470
|
raise NeatValueError("None of the classes implement Cognite Core concepts.")
|
|
2372
2471
|
return cognite_implements_concepts
|
|
2373
2472
|
|
|
2374
|
-
def
|
|
2375
|
-
self,
|
|
2376
|
-
) -> dict[
|
|
2377
|
-
view_ids = [
|
|
2473
|
+
def _get_views_by_concept(
|
|
2474
|
+
self, concepts: set[ConceptEntity], default_space: str, default_version: str
|
|
2475
|
+
) -> dict[ConceptEntity, View]:
|
|
2476
|
+
view_ids = [concept.as_view_entity(default_space, default_version).as_id() for concept in concepts]
|
|
2378
2477
|
views = self._client.loaders.views.retrieve(view_ids, include_ancestor=True, include_connected=True)
|
|
2379
|
-
return {
|
|
2478
|
+
return {ConceptEntity(prefix=view.space, suffix=view.external_id, version=view.version): view for view in views}
|