cognite-neat 0.104.0__py3-none-any.whl → 0.105.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
- cognite/neat/_client/_api/schema.py +2 -1
- cognite/neat/_client/data_classes/neat_sequence.py +261 -0
- cognite/neat/_client/data_classes/schema.py +5 -1
- cognite/neat/_client/testing.py +33 -0
- cognite/neat/_constants.py +56 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
- cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
- cognite/neat/_graph/transformers/__init__.py +3 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +2 -1
- cognite/neat/_graph/transformers/_value_type.py +72 -0
- cognite/neat/_issues/__init__.py +0 -2
- cognite/neat/_issues/_base.py +19 -35
- cognite/neat/_issues/warnings/__init__.py +4 -1
- cognite/neat/_issues/warnings/_general.py +7 -0
- cognite/neat/_issues/warnings/_resources.py +11 -0
- cognite/neat/_rules/exporters/_rules2dms.py +35 -1
- cognite/neat/_rules/exporters/_rules2excel.py +2 -2
- cognite/neat/_rules/importers/_dms2rules.py +66 -55
- cognite/neat/_rules/models/_base_rules.py +4 -1
- cognite/neat/_rules/models/entities/_wrapped.py +10 -5
- cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +271 -188
- cognite/neat/_rules/transformers/_mapping.py +75 -59
- cognite/neat/_rules/transformers/_verification.py +2 -3
- cognite/neat/_session/_inspect.py +3 -1
- cognite/neat/_session/_prepare.py +112 -24
- cognite/neat/_session/_read.py +33 -70
- cognite/neat/_session/_state.py +2 -2
- cognite/neat/_session/_to.py +2 -2
- cognite/neat/_store/_rules_store.py +4 -8
- cognite/neat/_utils/reader/_base.py +27 -0
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
- cognite_neat-0.105.0.dist-info/RECORD +179 -0
- {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
- cognite/neat/_app/api/__init__.py +0 -0
- cognite/neat/_app/api/asgi/metrics.py +0 -4
- cognite/neat/_app/api/configuration.py +0 -98
- cognite/neat/_app/api/context_manager/__init__.py +0 -3
- cognite/neat/_app/api/context_manager/manager.py +0 -16
- cognite/neat/_app/api/data_classes/__init__.py +0 -0
- cognite/neat/_app/api/data_classes/rest.py +0 -59
- cognite/neat/_app/api/explorer.py +0 -66
- cognite/neat/_app/api/routers/configuration.py +0 -25
- cognite/neat/_app/api/routers/crud.py +0 -102
- cognite/neat/_app/api/routers/metrics.py +0 -10
- cognite/neat/_app/api/routers/workflows.py +0 -224
- cognite/neat/_app/api/utils/__init__.py +0 -0
- cognite/neat/_app/api/utils/data_mapping.py +0 -17
- cognite/neat/_app/api/utils/logging.py +0 -26
- cognite/neat/_app/api/utils/query_templates.py +0 -92
- cognite/neat/_app/main.py +0 -17
- cognite/neat/_app/monitoring/__init__.py +0 -0
- cognite/neat/_app/monitoring/metrics.py +0 -69
- cognite/neat/_app/ui/index.html +0 -1
- cognite/neat/_app/ui/neat-app/.gitignore +0 -23
- cognite/neat/_app/ui/neat-app/README.md +0 -70
- cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
- cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
- cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
- cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
- cognite/neat/_app/ui/neat-app/build/index.html +0 -1
- cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
- cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
- cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
- cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
- cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
- cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
- cognite/neat/_app/ui/neat-app/package.json +0 -62
- cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
- cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
- cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
- cognite/neat/_app/ui/neat-app/public/index.html +0 -43
- cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
- cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
- cognite/neat/_app/ui/neat-app/src/App.css +0 -38
- cognite/neat/_app/ui/neat-app/src/App.js +0 -17
- cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
- cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
- cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
- cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
- cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
- cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
- cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
- cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
- cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
- cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
- cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
- cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
- cognite/neat/_app/ui/neat-app/src/index.css +0 -13
- cognite/neat/_app/ui/neat-app/src/index.js +0 -13
- cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
- cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
- cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
- cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
- cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
- cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
- cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
- cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
- cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
- cognite/neat/_workflows/__init__.py +0 -17
- cognite/neat/_workflows/base.py +0 -590
- cognite/neat/_workflows/cdf_store.py +0 -393
- cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
- cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
- cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
- cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
- cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
- cognite/neat/_workflows/manager.py +0 -292
- cognite/neat/_workflows/model.py +0 -203
- cognite/neat/_workflows/steps/__init__.py +0 -0
- cognite/neat/_workflows/steps/data_contracts.py +0 -109
- cognite/neat/_workflows/steps/lib/__init__.py +0 -0
- cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
- cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
- cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
- cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -323
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
- cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
- cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
- cognite/neat/_workflows/steps/step_model.py +0 -79
- cognite/neat/_workflows/steps_registry.py +0 -218
- cognite/neat/_workflows/tasks.py +0 -18
- cognite/neat/_workflows/triggers.py +0 -169
- cognite/neat/_workflows/utils.py +0 -19
- cognite_neat-0.104.0.dist-info/RECORD +0 -276
- {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from abc import ABC
|
|
3
3
|
from collections import defaultdict
|
|
4
|
-
from functools import cached_property
|
|
5
4
|
from typing import Any, ClassVar, Literal
|
|
6
5
|
|
|
7
6
|
from cognite.client import data_modeling as dm
|
|
8
7
|
|
|
9
8
|
from cognite.neat._client import NeatClient
|
|
10
9
|
from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
|
|
11
|
-
from cognite.neat._issues.warnings import
|
|
10
|
+
from cognite.neat._issues.warnings import PropertyOverwritingWarning
|
|
12
11
|
from cognite.neat._rules.models import DMSRules, SheetList
|
|
13
12
|
from cognite.neat._rules.models.data_types import Enum
|
|
14
|
-
from cognite.neat._rules.models.dms import DMSEnum, DMSProperty
|
|
15
|
-
from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
|
|
13
|
+
from cognite.neat._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
|
|
14
|
+
from cognite.neat._rules.models.entities import ClassEntity, ContainerEntity, ViewEntity
|
|
16
15
|
|
|
17
16
|
from ._base import RulesTransformer
|
|
18
17
|
|
|
@@ -104,11 +103,12 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
104
103
|
class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
105
104
|
"""Maps properties and classes using the given mapping.
|
|
106
105
|
|
|
107
|
-
**Note**: This transformer mutates the input rules.
|
|
108
|
-
|
|
109
106
|
Args:
|
|
110
|
-
mapping: The mapping to use.
|
|
111
|
-
|
|
107
|
+
mapping: The mapping to use represented as a DMSRules object.
|
|
108
|
+
data_type_conflict: How to handle data type conflicts. The default is "overwrite".
|
|
109
|
+
A data type conflicts occurs when the data type of a property in the mapping is different from the
|
|
110
|
+
data type of the property in the input rules. If "overwrite" the data type in the input rules is overwritten
|
|
111
|
+
with the data type in the mapping.
|
|
112
112
|
"""
|
|
113
113
|
|
|
114
114
|
_mapping_fields: ClassVar[frozenset[str]] = frozenset(
|
|
@@ -119,67 +119,83 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
|
119
119
|
self.mapping = mapping
|
|
120
120
|
self.data_type_conflict = data_type_conflict
|
|
121
121
|
|
|
122
|
-
@cached_property
|
|
123
|
-
def _view_by_entity_id(self) -> dict[str, DMSView]:
|
|
124
|
-
return {view.view.external_id: view for view in self.mapping.views}
|
|
125
|
-
|
|
126
|
-
@cached_property
|
|
127
|
-
def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
|
|
128
|
-
return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
|
|
129
|
-
|
|
130
122
|
def transform(self, rules: DMSRules) -> DMSRules:
|
|
131
123
|
if self.data_type_conflict != "overwrite":
|
|
132
124
|
raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
|
|
133
125
|
input_rules = rules
|
|
134
126
|
new_rules = input_rules.model_copy(deep=True)
|
|
135
127
|
|
|
136
|
-
for view in new_rules.views
|
|
137
|
-
|
|
138
|
-
|
|
128
|
+
views_by_external_id = {view.view.external_id: view for view in new_rules.views}
|
|
129
|
+
new_views: set[ViewEntity] = set()
|
|
130
|
+
for mapping_view in self.mapping.views:
|
|
131
|
+
if existing_view := views_by_external_id.get(mapping_view.view.external_id):
|
|
132
|
+
existing_view.implements = mapping_view.implements
|
|
133
|
+
else:
|
|
134
|
+
# We need to add all the views in the mapping that are not in the input rules.
|
|
135
|
+
# This is to ensure that all ValueTypes are present in the resulting rules.
|
|
136
|
+
# For example, if a property is a direct relation to an Equipment view, we need to add
|
|
137
|
+
# the Equipment view to the rules.
|
|
138
|
+
new_rules.views.append(mapping_view)
|
|
139
|
+
new_views.add(mapping_view.view)
|
|
139
140
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
if key not in self._property_by_view_property:
|
|
143
|
-
continue
|
|
144
|
-
mapping_prop = self._property_by_view_property[key]
|
|
145
|
-
to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
|
|
146
|
-
if conflicts and self.data_type_conflict == "overwrite":
|
|
147
|
-
warnings.warn(
|
|
148
|
-
PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
|
|
149
|
-
stacklevel=2,
|
|
150
|
-
)
|
|
151
|
-
elif conflicts:
|
|
152
|
-
raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
|
|
153
|
-
for field_name, value in to_overwrite.items():
|
|
154
|
-
setattr(prop, field_name, value)
|
|
155
|
-
prop.container = mapping_prop.container
|
|
156
|
-
prop.container_property = mapping_prop.container_property
|
|
157
|
-
|
|
158
|
-
# Add missing views used as value types
|
|
159
|
-
existing_views = {view.view for view in new_rules.views}
|
|
160
|
-
new_value_types = {
|
|
161
|
-
prop.value_type
|
|
162
|
-
for prop in new_rules.properties
|
|
163
|
-
if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
|
|
141
|
+
properties_by_view_property = {
|
|
142
|
+
(prop.view.external_id, prop.view_property): prop for prop in new_rules.properties
|
|
164
143
|
}
|
|
165
|
-
for
|
|
166
|
-
|
|
167
|
-
|
|
144
|
+
existing_enum_collections = {item.collection for item in new_rules.enum or []}
|
|
145
|
+
mapping_enums_by_collection: dict[ClassEntity, list[DMSEnum]] = defaultdict(list)
|
|
146
|
+
for item in self.mapping.enum or []:
|
|
147
|
+
mapping_enums_by_collection[item.collection].append(item)
|
|
148
|
+
existing_containers = {container.container for container in new_rules.containers or []}
|
|
149
|
+
mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
|
|
150
|
+
for mapping_prop in self.mapping.properties:
|
|
151
|
+
if existing_prop := properties_by_view_property.get(
|
|
152
|
+
(mapping_prop.view.external_id, mapping_prop.view_property)
|
|
153
|
+
):
|
|
154
|
+
to_overwrite, conflicts = self._find_overwrites(existing_prop, mapping_prop)
|
|
155
|
+
if conflicts and self.data_type_conflict == "overwrite":
|
|
156
|
+
warnings.warn(
|
|
157
|
+
PropertyOverwritingWarning(
|
|
158
|
+
existing_prop.view.as_id(), "view", existing_prop.view_property, tuple(conflicts)
|
|
159
|
+
),
|
|
160
|
+
stacklevel=2,
|
|
161
|
+
)
|
|
162
|
+
elif conflicts:
|
|
163
|
+
raise NeatValueError(
|
|
164
|
+
f"Conflicting properties for {existing_prop.view}.{existing_prop.view_property}: {conflicts}"
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
for field_name, value in to_overwrite.items():
|
|
168
|
+
setattr(existing_prop, field_name, value)
|
|
169
|
+
existing_prop.container = mapping_prop.container
|
|
170
|
+
existing_prop.container_property = mapping_prop.container_property
|
|
171
|
+
elif isinstance(mapping_prop.value_type, ViewEntity):
|
|
172
|
+
# All connections must be included in the rules. This is to update the
|
|
173
|
+
# ValueTypes of the implemented views.
|
|
174
|
+
new_rules.properties.append(mapping_prop)
|
|
175
|
+
elif mapping_prop.view in new_views:
|
|
176
|
+
# All properties of new views are included. Main motivation is GUIDs properties
|
|
177
|
+
new_rules.properties.append(mapping_prop)
|
|
168
178
|
else:
|
|
169
|
-
|
|
179
|
+
# Skipping mapped properties that are not in the input rules.
|
|
180
|
+
continue
|
|
170
181
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
182
|
+
if (
|
|
183
|
+
isinstance(mapping_prop.value_type, Enum)
|
|
184
|
+
and mapping_prop.value_type.collection not in existing_enum_collections
|
|
185
|
+
):
|
|
186
|
+
if not new_rules.enum:
|
|
187
|
+
new_rules.enum = SheetList[DMSEnum]([])
|
|
188
|
+
new_rules.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
|
|
189
|
+
|
|
190
|
+
if (
|
|
191
|
+
mapping_prop.container
|
|
192
|
+
and mapping_prop.container not in existing_containers
|
|
193
|
+
and (new_container := mapping_containers_by_id.get(mapping_prop.container))
|
|
194
|
+
):
|
|
195
|
+
# Mapping can include new containers for GUID properties
|
|
196
|
+
if not new_rules.containers:
|
|
197
|
+
new_rules.containers = SheetList[DMSContainer]([])
|
|
198
|
+
new_rules.containers.append(new_container)
|
|
183
199
|
|
|
184
200
|
return new_rules
|
|
185
201
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from abc import ABC
|
|
2
2
|
|
|
3
3
|
from cognite.neat._client import NeatClient
|
|
4
|
-
from cognite.neat._issues import
|
|
4
|
+
from cognite.neat._issues import MultiValueError, catch_issues
|
|
5
5
|
from cognite.neat._issues.errors import NeatTypeError, NeatValueError
|
|
6
6
|
from cognite.neat._rules._shared import (
|
|
7
7
|
ReadRules,
|
|
@@ -39,8 +39,7 @@ class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules
|
|
|
39
39
|
verified_rules: T_VerifiedRules | None = None
|
|
40
40
|
# We need to catch issues as we use the error args to provide extra context for the errors/warnings
|
|
41
41
|
# For example, which row in the spreadsheet the error occurred o
|
|
42
|
-
|
|
43
|
-
with catch_issues(issues, NeatError, NeatWarning, error_args) as _:
|
|
42
|
+
with catch_issues(error_args=error_args) as issues:
|
|
44
43
|
rules_cls = self._get_rules_cls(rules)
|
|
45
44
|
dumped = in_.dump()
|
|
46
45
|
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
@@ -59,7 +59,9 @@ class InspectAPI:
|
|
|
59
59
|
neat.inspect.properties
|
|
60
60
|
```
|
|
61
61
|
"""
|
|
62
|
-
|
|
62
|
+
df = self._state.rule_store.last_verified_rule.properties.to_pandas()
|
|
63
|
+
df.drop(columns=["neatId"], errors="ignore", inplace=True)
|
|
64
|
+
return df
|
|
63
65
|
|
|
64
66
|
|
|
65
67
|
@session_class_wrapper
|
|
@@ -8,8 +8,10 @@ from cognite.neat._client import NeatClient
|
|
|
8
8
|
from cognite.neat._constants import (
|
|
9
9
|
get_default_prefixes_and_namespaces,
|
|
10
10
|
)
|
|
11
|
+
from cognite.neat._graph import extractors
|
|
11
12
|
from cognite.neat._graph.transformers import (
|
|
12
13
|
AttachPropertyFromTargetToSource,
|
|
14
|
+
ConnectionToLiteral,
|
|
13
15
|
ConvertLiteral,
|
|
14
16
|
LiteralToEntity,
|
|
15
17
|
PruneDeadEndEdges,
|
|
@@ -20,14 +22,20 @@ from cognite.neat._graph.transformers import (
|
|
|
20
22
|
)
|
|
21
23
|
from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
22
24
|
from cognite.neat._issues import IssueList
|
|
25
|
+
from cognite.neat._issues.errors import NeatValueError
|
|
26
|
+
from cognite.neat._rules.models.dms import DMSValidation
|
|
23
27
|
from cognite.neat._rules.transformers import (
|
|
24
28
|
AddClassImplements,
|
|
25
29
|
IncludeReferenced,
|
|
26
30
|
PrefixEntities,
|
|
27
31
|
ReduceCogniteModel,
|
|
32
|
+
RulesTransformer,
|
|
28
33
|
ToCompliantEntities,
|
|
29
|
-
|
|
34
|
+
ToDataProductModel,
|
|
35
|
+
ToEnterpriseModel,
|
|
36
|
+
ToSolutionModel,
|
|
30
37
|
)
|
|
38
|
+
from cognite.neat._utils.text import humanize_collection
|
|
31
39
|
|
|
32
40
|
from ._state import SessionState
|
|
33
41
|
from .exceptions import NeatSessionError, session_class_wrapper
|
|
@@ -195,9 +203,11 @@ class InstancePrepareAPI:
|
|
|
195
203
|
neat.prepare.instances.make_connection_on_exact_match(source, target, connection)
|
|
196
204
|
```
|
|
197
205
|
"""
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
206
|
+
try:
|
|
207
|
+
subject_type, subject_predicate = self._get_type_and_property_uris(*source)
|
|
208
|
+
object_type, object_predicate = self._get_type_and_property_uris(*target)
|
|
209
|
+
except NeatValueError as e:
|
|
210
|
+
raise NeatSessionError(f"Cannot make connection: {e}") from None
|
|
201
211
|
|
|
202
212
|
transformer = MakeConnectionOnExactMatch(
|
|
203
213
|
subject_type,
|
|
@@ -215,17 +225,19 @@ class InstancePrepareAPI:
|
|
|
215
225
|
property_uri = self._state.instances.store.queries.property_uri(property_)
|
|
216
226
|
|
|
217
227
|
if not type_uri:
|
|
218
|
-
raise
|
|
228
|
+
raise NeatValueError(f"Type {type_} does not exist in the graph.")
|
|
219
229
|
elif len(type_uri) > 1:
|
|
220
|
-
raise
|
|
230
|
+
raise NeatValueError(f"{type_} has multiple ids found in the graph: {humanize_collection(type_uri)}.")
|
|
221
231
|
|
|
222
232
|
if not property_uri:
|
|
223
|
-
raise
|
|
233
|
+
raise NeatValueError(f"Property {property_} does not exist in the graph.")
|
|
224
234
|
elif len(type_uri) > 1:
|
|
225
|
-
raise
|
|
235
|
+
raise NeatValueError(
|
|
236
|
+
f"{property_} has multiple ids found in the graph: {humanize_collection(property_uri)}."
|
|
237
|
+
)
|
|
226
238
|
|
|
227
239
|
if not self._state.instances.store.queries.type_with_property(type_uri[0], property_uri[0]):
|
|
228
|
-
raise
|
|
240
|
+
raise NeatValueError(f"Property {property_} is not defined for type {type_}.")
|
|
229
241
|
return type_uri[0], property_uri[0]
|
|
230
242
|
|
|
231
243
|
def relationships_as_edges(self, min_relationship_types: int = 1, limit_per_type: int | None = None) -> None:
|
|
@@ -269,7 +281,10 @@ class InstancePrepareAPI:
|
|
|
269
281
|
```
|
|
270
282
|
|
|
271
283
|
"""
|
|
272
|
-
|
|
284
|
+
try:
|
|
285
|
+
subject_type, subject_predicate = self._get_type_and_property_uris(*source)
|
|
286
|
+
except NeatValueError as e:
|
|
287
|
+
raise NeatSessionError(f"Cannot convert data type: {e}") from None
|
|
273
288
|
|
|
274
289
|
transformer = ConvertLiteral(subject_type, subject_predicate, convert)
|
|
275
290
|
self._state.instances.store.transform(transformer)
|
|
@@ -294,13 +309,81 @@ class InstancePrepareAPI:
|
|
|
294
309
|
"""
|
|
295
310
|
subject_type: URIRef | None = None
|
|
296
311
|
if source[0] is not None:
|
|
297
|
-
|
|
312
|
+
try:
|
|
313
|
+
subject_type, subject_predicate = self._get_type_and_property_uris(*source) # type: ignore[arg-type, assignment]
|
|
314
|
+
except NeatValueError as e:
|
|
315
|
+
raise NeatSessionError(f"Cannot convert to type: {e}") from None
|
|
298
316
|
else:
|
|
299
317
|
subject_predicate = self._state.instances.store.queries.property_uri(source[1])[0]
|
|
300
318
|
|
|
301
319
|
transformer = LiteralToEntity(subject_type, subject_predicate, type, new_property)
|
|
302
320
|
self._state.instances.store.transform(transformer)
|
|
303
321
|
|
|
322
|
+
def connection_to_data_type(self, source: tuple[str | None, str]) -> None:
|
|
323
|
+
"""Converts a connection to a data type.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
source: The source of the conversion. A tuple of (type, property)
|
|
327
|
+
where property is the property that should be converted.
|
|
328
|
+
You can pass (None, property) to covert all properties with the given name.
|
|
329
|
+
|
|
330
|
+
Example:
|
|
331
|
+
|
|
332
|
+
Convert all properties 'labels' from a connection to a string:
|
|
333
|
+
|
|
334
|
+
```python
|
|
335
|
+
neat.prepare.instances.connection_to_data_type(
|
|
336
|
+
(None, "labels")
|
|
337
|
+
)
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
"""
|
|
341
|
+
subject_type: URIRef | None = None
|
|
342
|
+
if source[0] is not None:
|
|
343
|
+
try:
|
|
344
|
+
subject_type, subject_predicate = self._get_type_and_property_uris(*source) # type: ignore[arg-type, assignment]
|
|
345
|
+
except NeatValueError as e:
|
|
346
|
+
raise NeatSessionError(f"Cannot convert to data type: {e}") from None
|
|
347
|
+
else:
|
|
348
|
+
subject_predicate = self._state.instances.store.queries.property_uri(source[1])[0]
|
|
349
|
+
transformer = ConnectionToLiteral(subject_type, subject_predicate)
|
|
350
|
+
self._state.instances.store.transform(transformer)
|
|
351
|
+
|
|
352
|
+
def classic_to_core(self) -> None:
|
|
353
|
+
"""Prepares extracted CDF classic graph for the Core Data model.
|
|
354
|
+
|
|
355
|
+
!!! note "This method bundles several graph transformers which"
|
|
356
|
+
- Convert relationships to edges
|
|
357
|
+
- Convert TimeSeries.type from bool to enum
|
|
358
|
+
- Convert all properties 'source' to a connection to SourceSystem
|
|
359
|
+
- Convert all properties 'labels' from a connection to a string
|
|
360
|
+
|
|
361
|
+
Example:
|
|
362
|
+
Apply classic to core transformations:
|
|
363
|
+
```python
|
|
364
|
+
neat.prepare.instances.classic_to_core()
|
|
365
|
+
```
|
|
366
|
+
"""
|
|
367
|
+
self.relationships_as_edges()
|
|
368
|
+
self.convert_data_type(
|
|
369
|
+
("TimeSeries", "isString"), convert=lambda is_string: "string" if is_string else "numeric"
|
|
370
|
+
)
|
|
371
|
+
self.property_to_type((None, "source"), "SourceSystem", "name")
|
|
372
|
+
for type_ in [
|
|
373
|
+
extractors.EventsExtractor._default_rdf_type,
|
|
374
|
+
extractors.AssetsExtractor._default_rdf_type,
|
|
375
|
+
extractors.FilesExtractor._default_rdf_type,
|
|
376
|
+
]:
|
|
377
|
+
try:
|
|
378
|
+
subject_type, subject_predicate = self._get_type_and_property_uris(type_, "labels")
|
|
379
|
+
except NeatValueError:
|
|
380
|
+
# If the type_.labels does not exist, continue. This is not an error, it just means that the
|
|
381
|
+
# Labels is not used in the graph for that type.
|
|
382
|
+
continue
|
|
383
|
+
else:
|
|
384
|
+
transformer = ConnectionToLiteral(subject_type, subject_predicate)
|
|
385
|
+
self._state.instances.store.transform(transformer)
|
|
386
|
+
|
|
304
387
|
|
|
305
388
|
@session_class_wrapper
|
|
306
389
|
class DataModelPrepareAPI:
|
|
@@ -357,10 +440,9 @@ class DataModelPrepareAPI:
|
|
|
357
440
|
|
|
358
441
|
"""
|
|
359
442
|
return self._state.rule_transform(
|
|
360
|
-
|
|
443
|
+
ToEnterpriseModel(
|
|
361
444
|
new_model_id=data_model_id,
|
|
362
445
|
org_name=org_name,
|
|
363
|
-
type_="enterprise",
|
|
364
446
|
dummy_property=dummy_property,
|
|
365
447
|
move_connections=move_connections,
|
|
366
448
|
)
|
|
@@ -394,10 +476,9 @@ class DataModelPrepareAPI:
|
|
|
394
476
|
|
|
395
477
|
"""
|
|
396
478
|
return self._state.rule_transform(
|
|
397
|
-
|
|
479
|
+
ToSolutionModel(
|
|
398
480
|
new_model_id=data_model_id,
|
|
399
481
|
org_name=org_name,
|
|
400
|
-
type_="solution",
|
|
401
482
|
mode=mode,
|
|
402
483
|
dummy_property=dummy_property,
|
|
403
484
|
)
|
|
@@ -416,25 +497,32 @@ class DataModelPrepareAPI:
|
|
|
416
497
|
|
|
417
498
|
Args:
|
|
418
499
|
data_model_id: The data product data model id that is being created.
|
|
419
|
-
org_name: Organization name
|
|
500
|
+
org_name: Organization name used as prefix if the model is building on top of a Cognite Data Model.
|
|
420
501
|
include: The views to include in the data product data model. Can be either "same-space" or "all".
|
|
421
|
-
If you set same-space, only the views in the same space as the data model
|
|
502
|
+
If you set same-space, only the properties of the views in the same space as the data model
|
|
503
|
+
will be included.
|
|
422
504
|
"""
|
|
423
|
-
|
|
505
|
+
|
|
506
|
+
view_ids, container_ids = DMSValidation(
|
|
507
|
+
self._state.rule_store.last_verified_dms_rules
|
|
508
|
+
).imported_views_and_containers_ids()
|
|
509
|
+
transformers: list[RulesTransformer] = []
|
|
510
|
+
if (view_ids or container_ids) and self._client is None:
|
|
424
511
|
raise NeatSessionError(
|
|
425
512
|
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
426
513
|
"NEAT needs a client to lookup the definitions. "
|
|
427
514
|
"Please set the client in the session, NeatSession(client=client)."
|
|
428
515
|
)
|
|
429
|
-
|
|
430
|
-
IncludeReferenced(self._client, include_properties=True)
|
|
431
|
-
|
|
516
|
+
elif (view_ids or container_ids) and self._client:
|
|
517
|
+
transformers.append(IncludeReferenced(self._client, include_properties=True))
|
|
518
|
+
|
|
519
|
+
transformers.append(
|
|
520
|
+
ToDataProductModel(
|
|
432
521
|
new_model_id=data_model_id,
|
|
433
522
|
org_name=org_name,
|
|
434
|
-
type_="data_product",
|
|
435
523
|
include=include,
|
|
436
|
-
)
|
|
437
|
-
|
|
524
|
+
)
|
|
525
|
+
)
|
|
438
526
|
|
|
439
527
|
self._state.rule_transform(*transformers)
|
|
440
528
|
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -1,18 +1,16 @@
|
|
|
1
|
-
import tempfile
|
|
2
|
-
from pathlib import Path
|
|
3
1
|
from typing import Any, Literal, cast
|
|
4
2
|
|
|
5
3
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
6
4
|
|
|
7
5
|
from cognite.neat._client import NeatClient
|
|
8
|
-
from cognite.neat._constants import COGNITE_SPACES
|
|
9
6
|
from cognite.neat._graph import examples as instances_examples
|
|
10
7
|
from cognite.neat._graph import extractors
|
|
11
8
|
from cognite.neat._issues import IssueList
|
|
12
9
|
from cognite.neat._issues.errors import NeatValueError
|
|
10
|
+
from cognite.neat._issues.warnings import MissingCogniteClientWarning
|
|
13
11
|
from cognite.neat._rules import catalog, importers
|
|
14
12
|
from cognite.neat._rules.importers import BaseImporter
|
|
15
|
-
from cognite.neat._utils.reader import
|
|
13
|
+
from cognite.neat._utils.reader import NeatReader
|
|
16
14
|
|
|
17
15
|
from ._state import SessionState
|
|
18
16
|
from ._wizard import NeatObjectType, RDFFileType, XMLFileType, object_wizard, rdf_dm_wizard, xml_format_wizard
|
|
@@ -167,9 +165,8 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
167
165
|
io: file path to the Excel sheet
|
|
168
166
|
"""
|
|
169
167
|
reader = NeatReader.create(io)
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
return self._state.rule_import(importers.ExcelImporter(reader.path))
|
|
168
|
+
path = reader.materialize_path()
|
|
169
|
+
return self._state.rule_import(importers.ExcelImporter(path))
|
|
173
170
|
|
|
174
171
|
|
|
175
172
|
@session_class_wrapper
|
|
@@ -185,46 +182,32 @@ class ExcelExampleAPI(BaseReadAPI):
|
|
|
185
182
|
|
|
186
183
|
@session_class_wrapper
|
|
187
184
|
class YamlReadAPI(BaseReadAPI):
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
Args:
|
|
191
|
-
io: file path to the Yaml file in the case of "neat" yaml, or path to a zip folder or directory with several
|
|
192
|
-
Yaml files in the case of "toolkit".
|
|
185
|
+
def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
|
|
186
|
+
"""Reads a yaml with either neat rules, or several toolkit yaml files to import Data Model(s) into NeatSession.
|
|
193
187
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
"""
|
|
188
|
+
Args:
|
|
189
|
+
io: File path to the Yaml file in the case of "neat" yaml, or path to a zip folder or directory with several
|
|
190
|
+
Yaml files in the case of "toolkit".
|
|
191
|
+
format: The format of the yaml file(s). Can be either "neat" or "toolkit".
|
|
199
192
|
|
|
200
|
-
|
|
193
|
+
Example:
|
|
194
|
+
```python
|
|
195
|
+
neat.read.yaml("path_to_toolkit_yamls")
|
|
196
|
+
```
|
|
197
|
+
"""
|
|
201
198
|
reader = NeatReader.create(io)
|
|
202
|
-
|
|
203
|
-
raise NeatValueError("Only file paths are supported for YAML files")
|
|
199
|
+
path = reader.materialize_path()
|
|
204
200
|
importer: BaseImporter
|
|
205
201
|
if format == "neat":
|
|
206
|
-
importer = importers.YAMLImporter.from_file(
|
|
202
|
+
importer = importers.YAMLImporter.from_file(path, source_name=f"{reader!s}")
|
|
207
203
|
elif format == "toolkit":
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
ref_containers = dms_importer.root_schema.referenced_container()
|
|
216
|
-
if system_container_ids := [
|
|
217
|
-
container_id for container_id in ref_containers if container_id.space in COGNITE_SPACES
|
|
218
|
-
]:
|
|
219
|
-
if self._client is None:
|
|
220
|
-
raise NeatSessionError(
|
|
221
|
-
"No client provided. You are referencing Cognite containers in your data model, "
|
|
222
|
-
"NEAT needs a client to lookup the container definitions. "
|
|
223
|
-
"Please set the client in the session, NeatSession(client=client)."
|
|
224
|
-
)
|
|
225
|
-
system_containers = self._client.loaders.containers.retrieve(system_container_ids)
|
|
226
|
-
dms_importer.update_referenced_containers(system_containers)
|
|
227
|
-
|
|
204
|
+
dms_importer = importers.DMSImporter.from_path(path, self._client)
|
|
205
|
+
if dms_importer.issue_list.has_warning_type(MissingCogniteClientWarning):
|
|
206
|
+
raise NeatSessionError(
|
|
207
|
+
"No client provided. You are referencing Cognite containers in your data model, "
|
|
208
|
+
"NEAT needs a client to lookup the container definitions. "
|
|
209
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
210
|
+
)
|
|
228
211
|
importer = dms_importer
|
|
229
212
|
else:
|
|
230
213
|
raise NeatValueError(f"Unsupported YAML format: {format}")
|
|
@@ -251,17 +234,9 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
251
234
|
"""
|
|
252
235
|
|
|
253
236
|
def __call__(self, io: Any, type: str, primary_key: str) -> None:
|
|
254
|
-
reader = NeatReader.create(io)
|
|
255
|
-
if isinstance(reader, HttpFileReader):
|
|
256
|
-
path = Path(tempfile.gettempdir()).resolve() / reader.name
|
|
257
|
-
path.write_text(reader.read_text(), encoding="utf-8", newline="\n")
|
|
258
|
-
elif isinstance(reader, PathReader):
|
|
259
|
-
path = reader.path
|
|
260
|
-
else:
|
|
261
|
-
raise NeatValueError("Only file paths are supported for CSV files")
|
|
262
237
|
engine = import_engine()
|
|
263
238
|
engine.set.format = "csv"
|
|
264
|
-
engine.set.file =
|
|
239
|
+
engine.set.file = NeatReader.create(io).materialize_path()
|
|
265
240
|
engine.set.type = type
|
|
266
241
|
engine.set.primary_key = primary_key
|
|
267
242
|
extractor = engine.create_extractor()
|
|
@@ -283,14 +258,7 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
283
258
|
io: Any,
|
|
284
259
|
format: XMLFileType | None = None,
|
|
285
260
|
) -> None:
|
|
286
|
-
|
|
287
|
-
if isinstance(reader, GitHubReader):
|
|
288
|
-
path = Path(tempfile.gettempdir()).resolve() / reader.name
|
|
289
|
-
path.write_text(reader.read_text())
|
|
290
|
-
elif isinstance(reader, PathReader):
|
|
291
|
-
path = reader.path
|
|
292
|
-
else:
|
|
293
|
-
raise NeatValueError("Only file paths are supported for XML files")
|
|
261
|
+
path = NeatReader.create(io).materialize_path()
|
|
294
262
|
if format is None:
|
|
295
263
|
format = xml_format_wizard()
|
|
296
264
|
|
|
@@ -303,7 +271,7 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
303
271
|
else:
|
|
304
272
|
raise NeatValueError("Only support XML files of DEXPI format at the moment.")
|
|
305
273
|
|
|
306
|
-
def dexpi(self,
|
|
274
|
+
def dexpi(self, io: Any) -> None:
|
|
307
275
|
"""Reads a DEXPI file into the NeatSession.
|
|
308
276
|
|
|
309
277
|
Args:
|
|
@@ -314,13 +282,14 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
314
282
|
neat.read.xml.dexpi("url_or_path_to_dexpi_file")
|
|
315
283
|
```
|
|
316
284
|
"""
|
|
285
|
+
path = NeatReader.create(io).materialize_path()
|
|
317
286
|
engine = import_engine()
|
|
318
287
|
engine.set.format = "dexpi"
|
|
319
288
|
engine.set.file = path
|
|
320
289
|
extractor = engine.create_extractor()
|
|
321
290
|
self._state.instances.store.write(extractor)
|
|
322
291
|
|
|
323
|
-
def aml(self,
|
|
292
|
+
def aml(self, io: Any):
|
|
324
293
|
"""Reads an AML file into NeatSession.
|
|
325
294
|
|
|
326
295
|
Args:
|
|
@@ -331,6 +300,7 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
331
300
|
neat.read.xml.aml("url_or_path_to_aml_file")
|
|
332
301
|
```
|
|
333
302
|
"""
|
|
303
|
+
path = NeatReader.create(io).materialize_path()
|
|
334
304
|
engine = import_engine()
|
|
335
305
|
engine.set.format = "aml"
|
|
336
306
|
engine.set.file = path
|
|
@@ -362,9 +332,7 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
362
332
|
```
|
|
363
333
|
"""
|
|
364
334
|
reader = NeatReader.create(io)
|
|
365
|
-
|
|
366
|
-
raise NeatValueError("Only file paths are supported for RDF files")
|
|
367
|
-
importer = importers.OWLImporter.from_file(reader.path, source_name=f"file {reader!s}")
|
|
335
|
+
importer = importers.OWLImporter.from_file(reader.materialize_path(), source_name=f"file {reader!s}")
|
|
368
336
|
return self._state.rule_import(importer)
|
|
369
337
|
|
|
370
338
|
def imf(self, io: Any) -> IssueList:
|
|
@@ -379,9 +347,7 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
379
347
|
```
|
|
380
348
|
"""
|
|
381
349
|
reader = NeatReader.create(io)
|
|
382
|
-
|
|
383
|
-
raise NeatValueError("Only file paths are supported for RDF files")
|
|
384
|
-
importer = importers.IMFImporter.from_file(reader.path, source_name=f"file {reader!s}")
|
|
350
|
+
importer = importers.IMFImporter.from_file(reader.materialize_path(), source_name=f"file {reader!s}")
|
|
385
351
|
return self._state.rule_import(importer)
|
|
386
352
|
|
|
387
353
|
def __call__(
|
|
@@ -408,10 +374,7 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
408
374
|
|
|
409
375
|
elif type == "instances":
|
|
410
376
|
reader = NeatReader.create(io)
|
|
411
|
-
|
|
412
|
-
raise NeatValueError("Only file paths are supported for RDF files")
|
|
413
|
-
|
|
414
|
-
self._state.instances.store.write(extractors.RdfFileExtractor(reader.path))
|
|
377
|
+
self._state.instances.store.write(extractors.RdfFileExtractor(reader.materialize_path()))
|
|
415
378
|
return IssueList()
|
|
416
379
|
else:
|
|
417
380
|
raise NeatSessionError(f"Expected data model or instances, got {type}")
|
cognite/neat/_session/_state.py
CHANGED
|
@@ -4,7 +4,7 @@ from typing import Literal, cast
|
|
|
4
4
|
from cognite.neat._issues import IssueList
|
|
5
5
|
from cognite.neat._rules.importers import BaseImporter, InferenceImporter
|
|
6
6
|
from cognite.neat._rules.models import DMSRules, InformationRules
|
|
7
|
-
from cognite.neat._rules.transformers import RulesTransformer,
|
|
7
|
+
from cognite.neat._rules.transformers import RulesTransformer, ToExtensionModel
|
|
8
8
|
from cognite.neat._store import NeatGraphStore, NeatRulesStore
|
|
9
9
|
from cognite.neat._store._rules_store import ModelEntity
|
|
10
10
|
from cognite.neat._utils.rdf_ import uri_display_name
|
|
@@ -36,7 +36,7 @@ class SessionState:
|
|
|
36
36
|
f"Moving back {len(pruned)} {step_str} to the last {location}."
|
|
37
37
|
)
|
|
38
38
|
if (
|
|
39
|
-
any(isinstance(t,
|
|
39
|
+
any(isinstance(t, ToExtensionModel) for t in transformer)
|
|
40
40
|
and isinstance(self.rule_store.provenance[-1].target_entity, ModelEntity)
|
|
41
41
|
and isinstance(self.rule_store.provenance[-1].target_entity.result, DMSRules | InformationRules)
|
|
42
42
|
):
|