cognite-neat 0.103.1__py3-none-any.whl → 0.105.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
- cognite/neat/_client/_api/schema.py +2 -1
- cognite/neat/_client/data_classes/neat_sequence.py +261 -0
- cognite/neat/_client/data_classes/schema.py +5 -1
- cognite/neat/_client/testing.py +33 -0
- cognite/neat/_constants.py +56 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
- cognite/neat/_graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
- cognite/neat/_graph/transformers/__init__.py +3 -1
- cognite/neat/_graph/transformers/_base.py +109 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +6 -1
- cognite/neat/_graph/transformers/_prune_graph.py +103 -47
- cognite/neat/_graph/transformers/_rdfpath.py +41 -17
- cognite/neat/_graph/transformers/_value_type.py +188 -151
- cognite/neat/_issues/__init__.py +0 -2
- cognite/neat/_issues/_base.py +54 -43
- cognite/neat/_issues/warnings/__init__.py +4 -1
- cognite/neat/_issues/warnings/_general.py +7 -0
- cognite/neat/_issues/warnings/_resources.py +12 -1
- cognite/neat/_rules/_shared.py +18 -34
- cognite/neat/_rules/exporters/_base.py +28 -2
- cognite/neat/_rules/exporters/_rules2dms.py +39 -1
- cognite/neat/_rules/exporters/_rules2excel.py +13 -2
- cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
- cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
- cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
- cognite/neat/_rules/importers/_base.py +9 -0
- cognite/neat/_rules/importers/_dms2rules.py +80 -57
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
- cognite/neat/_rules/importers/_rdf/_base.py +10 -8
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
- cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
- cognite/neat/_rules/importers/_yaml2rules.py +21 -7
- cognite/neat/_rules/models/_base_input.py +1 -1
- cognite/neat/_rules/models/_base_rules.py +9 -1
- cognite/neat/_rules/models/dms/_rules.py +4 -0
- cognite/neat/_rules/models/dms/_rules_input.py +9 -0
- cognite/neat/_rules/models/entities/_wrapped.py +10 -5
- cognite/neat/_rules/models/information/_rules.py +4 -0
- cognite/neat/_rules/models/information/_rules_input.py +9 -0
- cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
- cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
- cognite/neat/_rules/transformers/__init__.py +13 -6
- cognite/neat/_rules/transformers/_base.py +41 -65
- cognite/neat/_rules/transformers/_converters.py +404 -234
- cognite/neat/_rules/transformers/_mapping.py +93 -72
- cognite/neat/_rules/transformers/_verification.py +50 -38
- cognite/neat/_session/_base.py +32 -121
- cognite/neat/_session/_inspect.py +5 -3
- cognite/neat/_session/_mapping.py +17 -105
- cognite/neat/_session/_prepare.py +138 -268
- cognite/neat/_session/_read.py +39 -195
- cognite/neat/_session/_set.py +6 -30
- cognite/neat/_session/_show.py +40 -21
- cognite/neat/_session/_state.py +49 -107
- cognite/neat/_session/_to.py +44 -33
- cognite/neat/_shared.py +23 -2
- cognite/neat/_store/_provenance.py +3 -82
- cognite/neat/_store/_rules_store.py +368 -10
- cognite/neat/_store/exceptions.py +23 -0
- cognite/neat/_utils/graph_transformations_report.py +36 -0
- cognite/neat/_utils/rdf_.py +8 -0
- cognite/neat/_utils/reader/_base.py +27 -0
- cognite/neat/_utils/spreadsheet.py +5 -4
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
- cognite_neat-0.105.0.dist-info/RECORD +179 -0
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
- cognite/neat/_app/api/__init__.py +0 -0
- cognite/neat/_app/api/asgi/metrics.py +0 -4
- cognite/neat/_app/api/configuration.py +0 -98
- cognite/neat/_app/api/context_manager/__init__.py +0 -3
- cognite/neat/_app/api/context_manager/manager.py +0 -16
- cognite/neat/_app/api/data_classes/__init__.py +0 -0
- cognite/neat/_app/api/data_classes/rest.py +0 -59
- cognite/neat/_app/api/explorer.py +0 -66
- cognite/neat/_app/api/routers/configuration.py +0 -25
- cognite/neat/_app/api/routers/crud.py +0 -102
- cognite/neat/_app/api/routers/metrics.py +0 -10
- cognite/neat/_app/api/routers/workflows.py +0 -224
- cognite/neat/_app/api/utils/__init__.py +0 -0
- cognite/neat/_app/api/utils/data_mapping.py +0 -17
- cognite/neat/_app/api/utils/logging.py +0 -26
- cognite/neat/_app/api/utils/query_templates.py +0 -92
- cognite/neat/_app/main.py +0 -17
- cognite/neat/_app/monitoring/__init__.py +0 -0
- cognite/neat/_app/monitoring/metrics.py +0 -69
- cognite/neat/_app/ui/index.html +0 -1
- cognite/neat/_app/ui/neat-app/.gitignore +0 -23
- cognite/neat/_app/ui/neat-app/README.md +0 -70
- cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
- cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
- cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
- cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
- cognite/neat/_app/ui/neat-app/build/index.html +0 -1
- cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
- cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
- cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
- cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
- cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
- cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
- cognite/neat/_app/ui/neat-app/package.json +0 -62
- cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
- cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
- cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
- cognite/neat/_app/ui/neat-app/public/index.html +0 -43
- cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
- cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
- cognite/neat/_app/ui/neat-app/src/App.css +0 -38
- cognite/neat/_app/ui/neat-app/src/App.js +0 -17
- cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
- cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
- cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
- cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
- cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
- cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
- cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
- cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
- cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
- cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
- cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
- cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
- cognite/neat/_app/ui/neat-app/src/index.css +0 -13
- cognite/neat/_app/ui/neat-app/src/index.js +0 -13
- cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
- cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
- cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
- cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
- cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
- cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
- cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
- cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
- cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
- cognite/neat/_rules/transformers/_pipelines.py +0 -70
- cognite/neat/_workflows/__init__.py +0 -17
- cognite/neat/_workflows/base.py +0 -590
- cognite/neat/_workflows/cdf_store.py +0 -393
- cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
- cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
- cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
- cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
- cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
- cognite/neat/_workflows/manager.py +0 -292
- cognite/neat/_workflows/model.py +0 -203
- cognite/neat/_workflows/steps/__init__.py +0 -0
- cognite/neat/_workflows/steps/data_contracts.py +0 -109
- cognite/neat/_workflows/steps/lib/__init__.py +0 -0
- cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
- cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
- cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
- cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -398
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
- cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
- cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
- cognite/neat/_workflows/steps/step_model.py +0 -79
- cognite/neat/_workflows/steps_registry.py +0 -218
- cognite/neat/_workflows/tasks.py +0 -18
- cognite/neat/_workflows/triggers.py +0 -169
- cognite/neat/_workflows/utils.py +0 -19
- cognite_neat-0.103.1.dist-info/RECORD +0 -275
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,19 +1,17 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from abc import ABC
|
|
3
3
|
from collections import defaultdict
|
|
4
|
-
from functools import cached_property
|
|
5
4
|
from typing import Any, ClassVar, Literal
|
|
6
5
|
|
|
7
6
|
from cognite.client import data_modeling as dm
|
|
8
7
|
|
|
9
8
|
from cognite.neat._client import NeatClient
|
|
10
9
|
from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
|
|
11
|
-
from cognite.neat._issues.warnings import
|
|
12
|
-
from cognite.neat._rules._shared import JustRules, OutRules
|
|
10
|
+
from cognite.neat._issues.warnings import PropertyOverwritingWarning
|
|
13
11
|
from cognite.neat._rules.models import DMSRules, SheetList
|
|
14
12
|
from cognite.neat._rules.models.data_types import Enum
|
|
15
|
-
from cognite.neat._rules.models.dms import DMSEnum, DMSProperty
|
|
16
|
-
from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
|
|
13
|
+
from cognite.neat._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
|
|
14
|
+
from cognite.neat._rules.models.entities import ClassEntity, ContainerEntity, ViewEntity
|
|
17
15
|
|
|
18
16
|
from ._base import RulesTransformer
|
|
19
17
|
|
|
@@ -55,8 +53,8 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
55
53
|
self.view_extension_mapping = view_extension_mapping
|
|
56
54
|
self.default_extension = default_extension
|
|
57
55
|
|
|
58
|
-
def transform(self, rules: DMSRules
|
|
59
|
-
solution: DMSRules =
|
|
56
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
57
|
+
solution: DMSRules = rules
|
|
60
58
|
view_by_external_id = {view.view.external_id: view for view in solution.views}
|
|
61
59
|
ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
|
|
62
60
|
|
|
@@ -99,17 +97,18 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
99
97
|
prop.container = ref_prop.container
|
|
100
98
|
prop.container_property = ref_prop.container_property
|
|
101
99
|
|
|
102
|
-
return
|
|
100
|
+
return solution
|
|
103
101
|
|
|
104
102
|
|
|
105
103
|
class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
106
104
|
"""Maps properties and classes using the given mapping.
|
|
107
105
|
|
|
108
|
-
**Note**: This transformer mutates the input rules.
|
|
109
|
-
|
|
110
106
|
Args:
|
|
111
|
-
mapping: The mapping to use.
|
|
112
|
-
|
|
107
|
+
mapping: The mapping to use represented as a DMSRules object.
|
|
108
|
+
data_type_conflict: How to handle data type conflicts. The default is "overwrite".
|
|
109
|
+
A data type conflicts occurs when the data type of a property in the mapping is different from the
|
|
110
|
+
data type of the property in the input rules. If "overwrite" the data type in the input rules is overwritten
|
|
111
|
+
with the data type in the mapping.
|
|
113
112
|
"""
|
|
114
113
|
|
|
115
114
|
_mapping_fields: ClassVar[frozenset[str]] = frozenset(
|
|
@@ -120,70 +119,85 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
|
120
119
|
self.mapping = mapping
|
|
121
120
|
self.data_type_conflict = data_type_conflict
|
|
122
121
|
|
|
123
|
-
|
|
124
|
-
def _view_by_entity_id(self) -> dict[str, DMSView]:
|
|
125
|
-
return {view.view.external_id: view for view in self.mapping.views}
|
|
126
|
-
|
|
127
|
-
@cached_property
|
|
128
|
-
def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
|
|
129
|
-
return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
|
|
130
|
-
|
|
131
|
-
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
122
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
132
123
|
if self.data_type_conflict != "overwrite":
|
|
133
124
|
raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
|
|
134
|
-
input_rules =
|
|
125
|
+
input_rules = rules
|
|
135
126
|
new_rules = input_rules.model_copy(deep=True)
|
|
136
|
-
new_rules.metadata.version += "_mapped"
|
|
137
127
|
|
|
138
|
-
for view in new_rules.views
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
key = (prop.view.external_id, prop.view_property)
|
|
144
|
-
if key not in self._property_by_view_property:
|
|
145
|
-
continue
|
|
146
|
-
mapping_prop = self._property_by_view_property[key]
|
|
147
|
-
to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
|
|
148
|
-
if conflicts and self.data_type_conflict == "overwrite":
|
|
149
|
-
warnings.warn(
|
|
150
|
-
PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
|
|
151
|
-
stacklevel=2,
|
|
152
|
-
)
|
|
153
|
-
elif conflicts:
|
|
154
|
-
raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
|
|
155
|
-
for field_name, value in to_overwrite.items():
|
|
156
|
-
setattr(prop, field_name, value)
|
|
157
|
-
prop.container = mapping_prop.container
|
|
158
|
-
prop.container_property = mapping_prop.container_property
|
|
159
|
-
|
|
160
|
-
# Add missing views used as value types
|
|
161
|
-
existing_views = {view.view for view in new_rules.views}
|
|
162
|
-
new_value_types = {
|
|
163
|
-
prop.value_type
|
|
164
|
-
for prop in new_rules.properties
|
|
165
|
-
if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
|
|
166
|
-
}
|
|
167
|
-
for new_value_type in new_value_types:
|
|
168
|
-
if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
|
|
169
|
-
new_rules.views.append(mapping_view)
|
|
128
|
+
views_by_external_id = {view.view.external_id: view for view in new_rules.views}
|
|
129
|
+
new_views: set[ViewEntity] = set()
|
|
130
|
+
for mapping_view in self.mapping.views:
|
|
131
|
+
if existing_view := views_by_external_id.get(mapping_view.view.external_id):
|
|
132
|
+
existing_view.implements = mapping_view.implements
|
|
170
133
|
else:
|
|
171
|
-
|
|
134
|
+
# We need to add all the views in the mapping that are not in the input rules.
|
|
135
|
+
# This is to ensure that all ValueTypes are present in the resulting rules.
|
|
136
|
+
# For example, if a property is a direct relation to an Equipment view, we need to add
|
|
137
|
+
# the Equipment view to the rules.
|
|
138
|
+
new_rules.views.append(mapping_view)
|
|
139
|
+
new_views.add(mapping_view.view)
|
|
172
140
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
new_enums = {
|
|
176
|
-
prop.value_type.collection
|
|
177
|
-
for prop in new_rules.properties
|
|
178
|
-
if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
|
|
141
|
+
properties_by_view_property = {
|
|
142
|
+
(prop.view.external_id, prop.view_property): prop for prop in new_rules.properties
|
|
179
143
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
144
|
+
existing_enum_collections = {item.collection for item in new_rules.enum or []}
|
|
145
|
+
mapping_enums_by_collection: dict[ClassEntity, list[DMSEnum]] = defaultdict(list)
|
|
146
|
+
for item in self.mapping.enum or []:
|
|
147
|
+
mapping_enums_by_collection[item.collection].append(item)
|
|
148
|
+
existing_containers = {container.container for container in new_rules.containers or []}
|
|
149
|
+
mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
|
|
150
|
+
for mapping_prop in self.mapping.properties:
|
|
151
|
+
if existing_prop := properties_by_view_property.get(
|
|
152
|
+
(mapping_prop.view.external_id, mapping_prop.view_property)
|
|
153
|
+
):
|
|
154
|
+
to_overwrite, conflicts = self._find_overwrites(existing_prop, mapping_prop)
|
|
155
|
+
if conflicts and self.data_type_conflict == "overwrite":
|
|
156
|
+
warnings.warn(
|
|
157
|
+
PropertyOverwritingWarning(
|
|
158
|
+
existing_prop.view.as_id(), "view", existing_prop.view_property, tuple(conflicts)
|
|
159
|
+
),
|
|
160
|
+
stacklevel=2,
|
|
161
|
+
)
|
|
162
|
+
elif conflicts:
|
|
163
|
+
raise NeatValueError(
|
|
164
|
+
f"Conflicting properties for {existing_prop.view}.{existing_prop.view_property}: {conflicts}"
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
for field_name, value in to_overwrite.items():
|
|
168
|
+
setattr(existing_prop, field_name, value)
|
|
169
|
+
existing_prop.container = mapping_prop.container
|
|
170
|
+
existing_prop.container_property = mapping_prop.container_property
|
|
171
|
+
elif isinstance(mapping_prop.value_type, ViewEntity):
|
|
172
|
+
# All connections must be included in the rules. This is to update the
|
|
173
|
+
# ValueTypes of the implemented views.
|
|
174
|
+
new_rules.properties.append(mapping_prop)
|
|
175
|
+
elif mapping_prop.view in new_views:
|
|
176
|
+
# All properties of new views are included. Main motivation is GUIDs properties
|
|
177
|
+
new_rules.properties.append(mapping_prop)
|
|
178
|
+
else:
|
|
179
|
+
# Skipping mapped properties that are not in the input rules.
|
|
180
|
+
continue
|
|
185
181
|
|
|
186
|
-
|
|
182
|
+
if (
|
|
183
|
+
isinstance(mapping_prop.value_type, Enum)
|
|
184
|
+
and mapping_prop.value_type.collection not in existing_enum_collections
|
|
185
|
+
):
|
|
186
|
+
if not new_rules.enum:
|
|
187
|
+
new_rules.enum = SheetList[DMSEnum]([])
|
|
188
|
+
new_rules.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
|
|
189
|
+
|
|
190
|
+
if (
|
|
191
|
+
mapping_prop.container
|
|
192
|
+
and mapping_prop.container not in existing_containers
|
|
193
|
+
and (new_container := mapping_containers_by_id.get(mapping_prop.container))
|
|
194
|
+
):
|
|
195
|
+
# Mapping can include new containers for GUID properties
|
|
196
|
+
if not new_rules.containers:
|
|
197
|
+
new_rules.containers = SheetList[DMSContainer]([])
|
|
198
|
+
new_rules.containers.append(new_container)
|
|
199
|
+
|
|
200
|
+
return new_rules
|
|
187
201
|
|
|
188
202
|
def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
|
|
189
203
|
"""Finds the properties that need to be overwritten and returns them.
|
|
@@ -212,6 +226,10 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
|
212
226
|
conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
|
|
213
227
|
return to_overwrite, conflicts
|
|
214
228
|
|
|
229
|
+
@property
|
|
230
|
+
def description(self) -> str:
|
|
231
|
+
return f"Mapping to {self.mapping.metadata.as_data_model_id()!r}."
|
|
232
|
+
|
|
215
233
|
|
|
216
234
|
class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
|
|
217
235
|
"""Looks up all view properties that map to the same container property,
|
|
@@ -221,10 +239,9 @@ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
|
|
|
221
239
|
def __init__(self, client: NeatClient | None = None) -> None:
|
|
222
240
|
self._client = client
|
|
223
241
|
|
|
224
|
-
def transform(self, rules: DMSRules
|
|
225
|
-
input_rules =
|
|
242
|
+
def transform(self, rules: DMSRules) -> DMSRules:
|
|
243
|
+
input_rules = rules
|
|
226
244
|
new_rules = input_rules.model_copy(deep=True)
|
|
227
|
-
new_rules.metadata.version += "_as_parent_name"
|
|
228
245
|
|
|
229
246
|
path_by_view = self._inheritance_path_by_view(new_rules)
|
|
230
247
|
view_by_container_property = self._view_by_container_properties(new_rules)
|
|
@@ -240,7 +257,7 @@ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
|
|
|
240
257
|
):
|
|
241
258
|
prop.view_property = parent_name
|
|
242
259
|
|
|
243
|
-
return
|
|
260
|
+
return new_rules
|
|
244
261
|
|
|
245
262
|
# Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
|
|
246
263
|
def _inheritance_path_by_view(self, rules: DMSRules) -> dict[ViewEntity, list[ViewEntity]]:
|
|
@@ -338,3 +355,7 @@ class AsParentPropertyId(RulesTransformer[DMSRules, DMSRules]):
|
|
|
338
355
|
_, prop_name = min(view_properties, key=lambda prop: len(path_by_view[prop[0]]))
|
|
339
356
|
parent_name_by_container_property[(container, container_property)] = prop_name
|
|
340
357
|
return parent_name_by_container_property
|
|
358
|
+
|
|
359
|
+
@property
|
|
360
|
+
def description(self) -> str:
|
|
361
|
+
return "Renaming property names to parent name"
|
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
from abc import ABC
|
|
2
|
-
from typing import Any, Literal
|
|
3
2
|
|
|
4
|
-
from cognite.neat.
|
|
5
|
-
from cognite.neat._issues
|
|
3
|
+
from cognite.neat._client import NeatClient
|
|
4
|
+
from cognite.neat._issues import MultiValueError, catch_issues
|
|
5
|
+
from cognite.neat._issues.errors import NeatTypeError, NeatValueError
|
|
6
6
|
from cognite.neat._rules._shared import (
|
|
7
|
-
InputRules,
|
|
8
|
-
MaybeRules,
|
|
9
|
-
OutRules,
|
|
10
7
|
ReadRules,
|
|
11
|
-
|
|
8
|
+
T_ReadInputRules,
|
|
12
9
|
T_VerifiedRules,
|
|
13
10
|
VerifiedRules,
|
|
14
11
|
)
|
|
@@ -24,78 +21,93 @@ from cognite.neat._rules.models.information import InformationValidation
|
|
|
24
21
|
from ._base import RulesTransformer
|
|
25
22
|
|
|
26
23
|
|
|
27
|
-
class VerificationTransformer(RulesTransformer[
|
|
24
|
+
class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules], ABC):
|
|
28
25
|
"""Base class for all verification transformers."""
|
|
29
26
|
|
|
30
27
|
_rules_cls: type[T_VerifiedRules]
|
|
31
28
|
_validation_cls: type
|
|
32
29
|
|
|
33
|
-
def __init__(self,
|
|
34
|
-
self.errors = errors
|
|
30
|
+
def __init__(self, validate: bool = True, client: NeatClient | None = None) -> None:
|
|
35
31
|
self.validate = validate
|
|
32
|
+
self._client = client
|
|
36
33
|
|
|
37
|
-
def transform(self, rules:
|
|
38
|
-
|
|
39
|
-
in_
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
error_args = rules.read_context
|
|
34
|
+
def transform(self, rules: T_ReadInputRules) -> T_VerifiedRules:
|
|
35
|
+
in_ = rules.rules
|
|
36
|
+
if in_ is None:
|
|
37
|
+
raise NeatValueError("Cannot verify rules. The reading of the rules failed.")
|
|
38
|
+
error_args = rules.read_context
|
|
43
39
|
verified_rules: T_VerifiedRules | None = None
|
|
44
|
-
|
|
45
|
-
|
|
40
|
+
# We need to catch issues as we use the error args to provide extra context for the errors/warnings
|
|
41
|
+
# For example, which row in the spreadsheet the error occurred o
|
|
42
|
+
with catch_issues(error_args=error_args) as issues:
|
|
43
|
+
rules_cls = self._get_rules_cls(rules)
|
|
46
44
|
dumped = in_.dump()
|
|
47
45
|
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
48
46
|
if self.validate:
|
|
49
47
|
validation_cls = self._get_validation_cls(verified_rules) # type: ignore[arg-type]
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
48
|
+
if issubclass(validation_cls, DMSValidation):
|
|
49
|
+
validation_issues = DMSValidation(verified_rules, self._client).validate() # type: ignore[arg-type]
|
|
50
|
+
elif issubclass(validation_cls, InformationValidation):
|
|
51
|
+
validation_issues = InformationValidation(verified_rules).validate() # type: ignore[arg-type]
|
|
52
|
+
else:
|
|
53
|
+
raise NeatValueError("Unsupported rule type")
|
|
54
|
+
|
|
55
|
+
# Need to trigger and raise such that the catch_issues can add the extra context
|
|
56
|
+
validation_issues.trigger_warnings()
|
|
55
57
|
if validation_issues.has_errors:
|
|
56
|
-
verified_rules = None
|
|
57
58
|
raise MultiValueError(validation_issues.errors)
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
60
|
+
# Raise issues which is expected to be handled outside of this method
|
|
61
|
+
issues.trigger_warnings()
|
|
62
|
+
if issues.has_errors:
|
|
63
|
+
raise MultiValueError(issues.errors)
|
|
64
|
+
if verified_rules is None:
|
|
65
|
+
raise NeatValueError("Rules were not verified")
|
|
66
|
+
return verified_rules
|
|
65
67
|
|
|
66
|
-
def _get_rules_cls(self, in_:
|
|
68
|
+
def _get_rules_cls(self, in_: T_ReadInputRules) -> type[T_VerifiedRules]:
|
|
67
69
|
return self._rules_cls
|
|
68
70
|
|
|
69
71
|
def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
|
|
70
72
|
return self._validation_cls
|
|
71
73
|
|
|
74
|
+
@property
|
|
75
|
+
def description(self) -> str:
|
|
76
|
+
return "Verify rules"
|
|
72
77
|
|
|
73
|
-
|
|
78
|
+
|
|
79
|
+
class VerifyDMSRules(VerificationTransformer[ReadRules[DMSInputRules], DMSRules]):
|
|
74
80
|
"""Class to verify DMS rules."""
|
|
75
81
|
|
|
76
82
|
_rules_cls = DMSRules
|
|
77
83
|
_validation_cls = DMSValidation
|
|
78
84
|
|
|
85
|
+
def transform(self, rules: ReadRules[DMSInputRules]) -> DMSRules:
|
|
86
|
+
return super().transform(rules)
|
|
87
|
+
|
|
79
88
|
|
|
80
|
-
class VerifyInformationRules(VerificationTransformer[InformationInputRules, InformationRules]):
|
|
89
|
+
class VerifyInformationRules(VerificationTransformer[ReadRules[InformationInputRules], InformationRules]):
|
|
81
90
|
"""Class to verify Information rules."""
|
|
82
91
|
|
|
83
92
|
_rules_cls = InformationRules
|
|
84
93
|
_validation_cls = InformationValidation
|
|
85
94
|
|
|
95
|
+
def transform(self, rules: ReadRules[InformationInputRules]) -> InformationRules:
|
|
96
|
+
return super().transform(rules)
|
|
86
97
|
|
|
87
|
-
|
|
98
|
+
|
|
99
|
+
class VerifyAnyRules(VerificationTransformer[T_ReadInputRules, VerifiedRules]):
|
|
88
100
|
"""Class to verify arbitrary rules"""
|
|
89
101
|
|
|
90
|
-
def _get_rules_cls(self, in_:
|
|
91
|
-
if isinstance(in_, InformationInputRules):
|
|
102
|
+
def _get_rules_cls(self, in_: T_ReadInputRules) -> type[VerifiedRules]:
|
|
103
|
+
if isinstance(in_.rules, InformationInputRules):
|
|
92
104
|
return InformationRules
|
|
93
|
-
elif isinstance(in_, DMSInputRules):
|
|
105
|
+
elif isinstance(in_.rules, DMSInputRules):
|
|
94
106
|
return DMSRules
|
|
95
107
|
else:
|
|
96
108
|
raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
|
|
97
109
|
|
|
98
|
-
def _get_validation_cls(self, rules:
|
|
110
|
+
def _get_validation_cls(self, rules: VerifiedRules) -> type:
|
|
99
111
|
if isinstance(rules, InformationRules):
|
|
100
112
|
return InformationValidation
|
|
101
113
|
elif isinstance(rules, DMSRules):
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -1,26 +1,17 @@
|
|
|
1
|
-
from
|
|
2
|
-
from typing import Literal, cast
|
|
1
|
+
from typing import Literal
|
|
3
2
|
|
|
4
3
|
from cognite.client import CogniteClient
|
|
5
4
|
from cognite.client import data_modeling as dm
|
|
6
5
|
|
|
7
6
|
from cognite.neat import _version
|
|
8
7
|
from cognite.neat._client import NeatClient
|
|
9
|
-
from cognite.neat._issues import IssueList
|
|
8
|
+
from cognite.neat._issues import IssueList
|
|
10
9
|
from cognite.neat._issues.errors import RegexViolationError
|
|
11
10
|
from cognite.neat._rules import importers
|
|
12
|
-
from cognite.neat._rules.
|
|
13
|
-
from cognite.neat._rules.models import DMSRules
|
|
14
|
-
from cognite.neat._rules.models.dms import DMSValidation
|
|
15
|
-
from cognite.neat._rules.models.information import InformationValidation
|
|
11
|
+
from cognite.neat._rules.models._base_input import InputRules
|
|
16
12
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
17
|
-
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
18
13
|
from cognite.neat._rules.transformers import ConvertToRules, InformationToDMS, VerifyAnyRules
|
|
19
14
|
from cognite.neat._rules.transformers._converters import ConversionTransformer
|
|
20
|
-
from cognite.neat._store._provenance import (
|
|
21
|
-
INSTANCES_ENTITY,
|
|
22
|
-
Change,
|
|
23
|
-
)
|
|
24
15
|
|
|
25
16
|
from ._collector import _COLLECTOR, Collector
|
|
26
17
|
from ._drop import DropAPI
|
|
@@ -128,47 +119,16 @@ class NeatSession:
|
|
|
128
119
|
neat.verify()
|
|
129
120
|
```
|
|
130
121
|
"""
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
elif isinstance(output.rules, InformationRules):
|
|
140
|
-
issues = InformationValidation(output.rules).validate()
|
|
141
|
-
else:
|
|
142
|
-
raise NeatSessionError("Unsupported rule type")
|
|
143
|
-
if issues.has_errors:
|
|
144
|
-
# This is up for discussion, but I think we should not return rules that
|
|
145
|
-
# only pass the verification but not the validation.
|
|
146
|
-
output.rules = None
|
|
147
|
-
output.issues.extend(issues)
|
|
148
|
-
|
|
149
|
-
end = datetime.now(timezone.utc)
|
|
150
|
-
|
|
151
|
-
if output.rules:
|
|
152
|
-
change = Change.from_rules_activity(
|
|
153
|
-
output.rules,
|
|
154
|
-
transformer.agent,
|
|
155
|
-
start,
|
|
156
|
-
end,
|
|
157
|
-
f"Verified data model {source_id} as {output.rules.metadata.identifier}",
|
|
158
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
159
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
160
|
-
)
|
|
161
|
-
|
|
162
|
-
self._state.data_model.write(output.rules, change)
|
|
163
|
-
|
|
164
|
-
if isinstance(output.rules, InformationRules):
|
|
165
|
-
self._state.instances.store.add_rules(output.rules)
|
|
166
|
-
|
|
167
|
-
output.issues.action = "verify"
|
|
168
|
-
self._state.data_model.issue_lists.append(output.issues)
|
|
169
|
-
if output.issues:
|
|
122
|
+
transformer = VerifyAnyRules(validate=True, client=self._client) # type: ignore[var-annotated]
|
|
123
|
+
issues = self._state.rule_transform(transformer)
|
|
124
|
+
if not issues.has_errors:
|
|
125
|
+
rules = self._state.rule_store.last_verified_rule
|
|
126
|
+
if isinstance(rules, InformationRules):
|
|
127
|
+
self._state.instances.store.add_rules(rules)
|
|
128
|
+
|
|
129
|
+
if issues:
|
|
170
130
|
print("You can inspect the issues with the .inspect.issues(...) method.")
|
|
171
|
-
return
|
|
131
|
+
return issues
|
|
172
132
|
|
|
173
133
|
def convert(
|
|
174
134
|
self, target: Literal["dms", "information"], mode: Literal["edge_properties"] | None = None
|
|
@@ -192,43 +152,17 @@ class NeatSession:
|
|
|
192
152
|
neat.convert(target="information")
|
|
193
153
|
```
|
|
194
154
|
"""
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
converter = ConvertToRules(InformationRules)
|
|
207
|
-
converted_rules = converter.transform(dms_rules).rules
|
|
208
|
-
else:
|
|
209
|
-
# Session errors are not caught by the catch_issues context manager
|
|
210
|
-
raise NeatSessionError(f"Target {target} not supported.")
|
|
211
|
-
|
|
212
|
-
end = datetime.now(timezone.utc)
|
|
213
|
-
if issues:
|
|
214
|
-
self._state.data_model.issue_lists.append(issues)
|
|
215
|
-
|
|
216
|
-
if converted_rules is not None and converter is not None:
|
|
217
|
-
# Provenance
|
|
218
|
-
change = Change.from_rules_activity(
|
|
219
|
-
converted_rules,
|
|
220
|
-
converter.agent,
|
|
221
|
-
start,
|
|
222
|
-
end,
|
|
223
|
-
f"Converted data model {source_id} to {converted_rules.metadata.identifier}",
|
|
224
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
225
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
226
|
-
)
|
|
227
|
-
|
|
228
|
-
self._state.data_model.write(converted_rules, change)
|
|
229
|
-
|
|
230
|
-
if self._verbose and not issues.has_errors:
|
|
231
|
-
print(f"Rules converted to {target}")
|
|
155
|
+
converter: ConversionTransformer
|
|
156
|
+
if target == "dms":
|
|
157
|
+
converter = InformationToDMS(mode=mode)
|
|
158
|
+
elif target == "information":
|
|
159
|
+
converter = ConvertToRules(InformationRules)
|
|
160
|
+
else:
|
|
161
|
+
raise NeatSessionError(f"Target {target} not supported.")
|
|
162
|
+
issues = self._state.rule_transform(converter)
|
|
163
|
+
|
|
164
|
+
if self._verbose and not issues.has_errors:
|
|
165
|
+
print(f"Rules converted to {target}")
|
|
232
166
|
else:
|
|
233
167
|
print("Conversion failed.")
|
|
234
168
|
if issues:
|
|
@@ -263,53 +197,30 @@ class NeatSession:
|
|
|
263
197
|
```
|
|
264
198
|
"""
|
|
265
199
|
model_id = dm.DataModelId.load(model_id)
|
|
266
|
-
|
|
267
|
-
start = datetime.now(timezone.utc)
|
|
268
200
|
importer = importers.InferenceImporter.from_graph_store(
|
|
269
201
|
store=self._state.instances.store,
|
|
270
202
|
max_number_of_instance=max_number_of_instance,
|
|
203
|
+
data_model_id=model_id,
|
|
271
204
|
)
|
|
272
|
-
|
|
273
|
-
end = datetime.now(timezone.utc)
|
|
274
|
-
|
|
275
|
-
if model_id.space:
|
|
276
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.space = model_id.space
|
|
277
|
-
if model_id.external_id:
|
|
278
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.external_id = model_id.external_id
|
|
279
|
-
|
|
280
|
-
if model_id.version:
|
|
281
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.version = model_id.version
|
|
282
|
-
|
|
283
|
-
# Provenance
|
|
284
|
-
change = Change.from_rules_activity(
|
|
285
|
-
inferred_rules,
|
|
286
|
-
importer.agent,
|
|
287
|
-
start,
|
|
288
|
-
end,
|
|
289
|
-
"Inferred data model",
|
|
290
|
-
INSTANCES_ENTITY,
|
|
291
|
-
)
|
|
292
|
-
|
|
293
|
-
self._state.data_model.write(inferred_rules, change)
|
|
294
|
-
return inferred_rules.issues
|
|
205
|
+
return self._state.rule_import(importer)
|
|
295
206
|
|
|
296
207
|
def _repr_html_(self) -> str:
|
|
297
208
|
state = self._state
|
|
298
209
|
if (
|
|
299
210
|
not state.instances.has_store
|
|
300
|
-
and not state.
|
|
301
|
-
and not state.
|
|
211
|
+
and not state.rule_store.has_unverified_rules
|
|
212
|
+
and not state.rule_store.has_verified_rules
|
|
302
213
|
):
|
|
303
214
|
return "<strong>Empty session</strong>. Get started by reading something with the <em>.read</em> attribute."
|
|
304
215
|
|
|
305
216
|
output = []
|
|
306
217
|
|
|
307
|
-
if state.
|
|
308
|
-
rules:
|
|
309
|
-
output.append(f"<H2>Unverified Data Model</H2><br />{rules.
|
|
218
|
+
if state.rule_store.has_unverified_rules and not state.rule_store.has_verified_rules:
|
|
219
|
+
rules: InputRules = state.rule_store.last_unverified_rule
|
|
220
|
+
output.append(f"<H2>Unverified Data Model</H2><br />{rules._repr_html_()}") # type: ignore
|
|
310
221
|
|
|
311
|
-
if state.
|
|
312
|
-
output.append(f"<H2>Verified Data Model</H2><br />{state.
|
|
222
|
+
if state.rule_store.has_verified_rules:
|
|
223
|
+
output.append(f"<H2>Verified Data Model</H2><br />{state.rule_store.last_verified_rule._repr_html_()}") # type: ignore
|
|
313
224
|
|
|
314
225
|
if state.instances.has_store:
|
|
315
226
|
output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
|
|
@@ -59,7 +59,9 @@ class InspectAPI:
|
|
|
59
59
|
neat.inspect.properties
|
|
60
60
|
```
|
|
61
61
|
"""
|
|
62
|
-
|
|
62
|
+
df = self._state.rule_store.last_verified_rule.properties.to_pandas()
|
|
63
|
+
df.drop(columns=["neatId"], errors="ignore", inplace=True)
|
|
64
|
+
return df
|
|
63
65
|
|
|
64
66
|
|
|
65
67
|
@session_class_wrapper
|
|
@@ -89,7 +91,7 @@ class InspectIssues:
|
|
|
89
91
|
return_dataframe: bool = (False if IN_NOTEBOOK else True), # type: ignore[assignment]
|
|
90
92
|
) -> pd.DataFrame | None:
|
|
91
93
|
"""Returns the issues of the current data model."""
|
|
92
|
-
issues = self._state.
|
|
94
|
+
issues = self._state.rule_store.last_issues
|
|
93
95
|
if not issues:
|
|
94
96
|
self._print("No issues found.")
|
|
95
97
|
|
|
@@ -141,7 +143,7 @@ class InspectOutcome:
|
|
|
141
143
|
"""
|
|
142
144
|
|
|
143
145
|
def __init__(self, state: SessionState) -> None:
|
|
144
|
-
self.data_model = InspectUploadOutcome(lambda: state.
|
|
146
|
+
self.data_model = InspectUploadOutcome(lambda: state.rule_store.last_outcome)
|
|
145
147
|
self.instances = InspectUploadOutcome(lambda: state.instances.last_outcome)
|
|
146
148
|
|
|
147
149
|
|