cognite-neat 0.103.0__py3-none-any.whl → 0.104.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/_graph/transformers/_base.py +109 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +4 -0
- cognite/neat/_graph/transformers/_prune_graph.py +103 -47
- cognite/neat/_graph/transformers/_rdfpath.py +41 -17
- cognite/neat/_graph/transformers/_value_type.py +119 -154
- cognite/neat/_issues/_base.py +35 -8
- cognite/neat/_issues/warnings/_resources.py +1 -1
- cognite/neat/_rules/_shared.py +18 -34
- cognite/neat/_rules/exporters/_base.py +28 -2
- cognite/neat/_rules/exporters/_rules2dms.py +4 -0
- cognite/neat/_rules/exporters/_rules2excel.py +11 -0
- cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
- cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
- cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
- cognite/neat/_rules/importers/_base.py +9 -0
- cognite/neat/_rules/importers/_dms2rules.py +17 -5
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
- cognite/neat/_rules/importers/_rdf/_base.py +10 -8
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
- cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
- cognite/neat/_rules/importers/_yaml2rules.py +21 -7
- cognite/neat/_rules/models/_base_input.py +1 -1
- cognite/neat/_rules/models/_base_rules.py +5 -0
- cognite/neat/_rules/models/dms/_rules.py +4 -0
- cognite/neat/_rules/models/dms/_rules_input.py +9 -0
- cognite/neat/_rules/models/dms/_validation.py +2 -0
- cognite/neat/_rules/models/entities/_single_value.py +25 -5
- cognite/neat/_rules/models/information/_rules.py +4 -0
- cognite/neat/_rules/models/information/_rules_input.py +9 -0
- cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
- cognite/neat/_rules/transformers/__init__.py +5 -4
- cognite/neat/_rules/transformers/_base.py +41 -65
- cognite/neat/_rules/transformers/_converters.py +149 -62
- cognite/neat/_rules/transformers/_mapping.py +17 -12
- cognite/neat/_rules/transformers/_verification.py +50 -37
- cognite/neat/_session/_base.py +32 -121
- cognite/neat/_session/_inspect.py +3 -3
- cognite/neat/_session/_mapping.py +17 -105
- cognite/neat/_session/_prepare.py +36 -254
- cognite/neat/_session/_read.py +11 -130
- cognite/neat/_session/_set.py +6 -30
- cognite/neat/_session/_show.py +49 -30
- cognite/neat/_session/_state.py +49 -107
- cognite/neat/_session/_to.py +42 -31
- cognite/neat/_shared.py +23 -2
- cognite/neat/_store/_provenance.py +3 -82
- cognite/neat/_store/_rules_store.py +372 -10
- cognite/neat/_store/exceptions.py +23 -0
- cognite/neat/_utils/graph_transformations_report.py +36 -0
- cognite/neat/_utils/io_.py +11 -0
- cognite/neat/_utils/rdf_.py +8 -0
- cognite/neat/_utils/spreadsheet.py +5 -4
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +7 -7
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +24 -99
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/RECORD +63 -61
- cognite/neat/_rules/transformers/_pipelines.py +0 -70
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/entry_points.txt +0 -0
|
@@ -19,7 +19,7 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
19
19
|
from cognite.client.utils import ms_to_datetime
|
|
20
20
|
|
|
21
21
|
from cognite.neat._client import NeatClient
|
|
22
|
-
from cognite.neat._issues import IssueList, NeatIssue
|
|
22
|
+
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
|
|
23
23
|
from cognite.neat._issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
|
|
24
24
|
from cognite.neat._issues.warnings import (
|
|
25
25
|
PropertyNotFoundWarning,
|
|
@@ -89,6 +89,14 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
89
89
|
continue
|
|
90
90
|
self._all_containers_by_id[container.as_id()] = container
|
|
91
91
|
|
|
92
|
+
@property
|
|
93
|
+
def description(self) -> str:
|
|
94
|
+
if self.root_schema.data_model is not None:
|
|
95
|
+
identifier = f"{self.root_schema.data_model.as_id().as_tuple()!s}"
|
|
96
|
+
else:
|
|
97
|
+
identifier = "Unknown"
|
|
98
|
+
return f"DMS Data model {identifier} read as unverified data model"
|
|
99
|
+
|
|
92
100
|
@classmethod
|
|
93
101
|
def from_data_model_id(
|
|
94
102
|
cls,
|
|
@@ -197,12 +205,14 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
197
205
|
if self.issue_list.has_errors:
|
|
198
206
|
# In case there were errors during the import, the to_rules method will return None
|
|
199
207
|
self._end = datetime.now(timezone.utc)
|
|
200
|
-
|
|
208
|
+
self.issue_list.trigger_warnings()
|
|
209
|
+
raise MultiValueError(self.issue_list.errors)
|
|
201
210
|
|
|
202
211
|
if not self.root_schema.data_model:
|
|
203
212
|
self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
|
|
204
213
|
self._end = datetime.now(timezone.utc)
|
|
205
|
-
|
|
214
|
+
self.issue_list.trigger_warnings()
|
|
215
|
+
raise MultiValueError(self.issue_list.errors)
|
|
206
216
|
|
|
207
217
|
model = self.root_schema.data_model
|
|
208
218
|
|
|
@@ -213,8 +223,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
213
223
|
)
|
|
214
224
|
|
|
215
225
|
self._end = datetime.now(timezone.utc)
|
|
216
|
-
|
|
217
|
-
|
|
226
|
+
self.issue_list.trigger_warnings()
|
|
227
|
+
if self.issue_list.has_errors:
|
|
228
|
+
raise MultiValueError(self.issue_list.errors)
|
|
229
|
+
return ReadRules(user_rules, {})
|
|
218
230
|
|
|
219
231
|
def _create_rule_components(
|
|
220
232
|
self,
|
|
@@ -5,7 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from pydantic import ValidationError
|
|
7
7
|
|
|
8
|
-
from cognite.neat._issues import IssueList, NeatIssue
|
|
8
|
+
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
|
|
9
9
|
from cognite.neat._issues.warnings import (
|
|
10
10
|
FileItemNotSupportedWarning,
|
|
11
11
|
FileMissingRequiredFieldWarning,
|
|
@@ -144,5 +144,8 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
|
144
144
|
properties=converter.properties,
|
|
145
145
|
classes=converter.classes,
|
|
146
146
|
)
|
|
147
|
+
converter.issues.trigger_warnings()
|
|
148
|
+
if converter.issues.has_errors:
|
|
149
|
+
raise MultiValueError(converter.issues.errors)
|
|
147
150
|
|
|
148
|
-
return ReadRules(rules,
|
|
151
|
+
return ReadRules(rules, {})
|
|
@@ -5,7 +5,7 @@ from cognite.client import data_modeling as dm
|
|
|
5
5
|
from rdflib import Graph, Namespace, URIRef
|
|
6
6
|
|
|
7
7
|
from cognite.neat._constants import get_default_prefixes_and_namespaces
|
|
8
|
-
from cognite.neat._issues import IssueList
|
|
8
|
+
from cognite.neat._issues import IssueList, MultiValueError
|
|
9
9
|
from cognite.neat._issues.errors import FileReadError
|
|
10
10
|
from cognite.neat._issues.errors._general import NeatValueError
|
|
11
11
|
from cognite.neat._rules._shared import ReadRules
|
|
@@ -13,9 +13,7 @@ from cognite.neat._rules.importers._base import BaseImporter
|
|
|
13
13
|
from cognite.neat._rules.models._base_rules import RoleTypes
|
|
14
14
|
from cognite.neat._rules.models.data_types import AnyURI
|
|
15
15
|
from cognite.neat._rules.models.entities import UnknownEntity
|
|
16
|
-
from cognite.neat._rules.models.information import
|
|
17
|
-
InformationInputRules,
|
|
18
|
-
)
|
|
16
|
+
from cognite.neat._rules.models.information import InformationInputRules
|
|
19
17
|
from cognite.neat._store import NeatGraphStore
|
|
20
18
|
from cognite.neat._utils.rdf_ import get_namespace
|
|
21
19
|
|
|
@@ -50,6 +48,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
|
|
|
50
48
|
max_number_of_instance: int,
|
|
51
49
|
non_existing_node_type: UnknownEntity | AnyURI,
|
|
52
50
|
language: str,
|
|
51
|
+
source_name: str = "Unknown",
|
|
53
52
|
) -> None:
|
|
54
53
|
self.issue_list = issue_list
|
|
55
54
|
self.graph = graph
|
|
@@ -60,6 +59,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
|
|
|
60
59
|
self.max_number_of_instance = max_number_of_instance
|
|
61
60
|
self.non_existing_node_type = non_existing_node_type
|
|
62
61
|
self.language = language
|
|
62
|
+
self.source_name = source_name
|
|
63
63
|
|
|
64
64
|
@classmethod
|
|
65
65
|
def from_graph_store(
|
|
@@ -87,6 +87,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
|
|
|
87
87
|
max_number_of_instance: int = -1,
|
|
88
88
|
non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
|
|
89
89
|
language: str = "en",
|
|
90
|
+
source_name: str = "Unknown",
|
|
90
91
|
):
|
|
91
92
|
issue_list = IssueList(title=f"{cls.__name__} issues")
|
|
92
93
|
|
|
@@ -107,6 +108,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
|
|
|
107
108
|
max_number_of_instance=max_number_of_instance,
|
|
108
109
|
non_existing_node_type=non_existing_node_type,
|
|
109
110
|
language=language,
|
|
111
|
+
source_name=source_name,
|
|
110
112
|
)
|
|
111
113
|
|
|
112
114
|
def to_rules(
|
|
@@ -115,16 +117,16 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
|
|
|
115
117
|
"""
|
|
116
118
|
Creates `Rules` object from the data for target role.
|
|
117
119
|
"""
|
|
118
|
-
|
|
119
120
|
if self.issue_list.has_errors:
|
|
120
121
|
# In case there were errors during the import, the to_rules method will return None
|
|
121
|
-
|
|
122
|
+
self.issue_list.trigger_warnings()
|
|
123
|
+
raise MultiValueError(self.issue_list.errors)
|
|
122
124
|
|
|
123
125
|
rules_dict = self._to_rules_components()
|
|
124
126
|
|
|
125
127
|
rules = InformationInputRules.load(rules_dict)
|
|
126
|
-
|
|
127
|
-
return ReadRules(rules,
|
|
128
|
+
self.issue_list.trigger_warnings()
|
|
129
|
+
return ReadRules(rules, {})
|
|
128
130
|
|
|
129
131
|
def _to_rules_components(self) -> dict:
|
|
130
132
|
raise NotImplementedError()
|
|
@@ -74,6 +74,10 @@ PROPERTIES_QUERY = """
|
|
|
74
74
|
class IMFImporter(BaseRDFImporter):
|
|
75
75
|
"""Convert IMF Types provided as SHACL shapes to Input Rules."""
|
|
76
76
|
|
|
77
|
+
@property
|
|
78
|
+
def description(self) -> str:
|
|
79
|
+
return f"IMF Types {self.source_name} read as unverified data model"
|
|
80
|
+
|
|
77
81
|
def _to_rules_components(
|
|
78
82
|
self,
|
|
79
83
|
) -> dict:
|
|
@@ -16,6 +16,7 @@ from cognite.neat._rules.models.information import (
|
|
|
16
16
|
InformationMetadata,
|
|
17
17
|
)
|
|
18
18
|
from cognite.neat._store import NeatGraphStore
|
|
19
|
+
from cognite.neat._store._provenance import INSTANCES_ENTITY
|
|
19
20
|
from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
|
|
20
21
|
|
|
21
22
|
from ._base import DEFAULT_NON_EXISTING_NODE_TYPE, BaseRDFImporter
|
|
@@ -94,6 +95,7 @@ class InferenceImporter(BaseRDFImporter):
|
|
|
94
95
|
max_number_of_instance: int = -1,
|
|
95
96
|
non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
|
|
96
97
|
language: str = "en",
|
|
98
|
+
source_name: str = "Unknown",
|
|
97
99
|
) -> "InferenceImporter":
|
|
98
100
|
return super().from_file(
|
|
99
101
|
filepath,
|
|
@@ -101,6 +103,7 @@ class InferenceImporter(BaseRDFImporter):
|
|
|
101
103
|
max_number_of_instance,
|
|
102
104
|
non_existing_node_type,
|
|
103
105
|
language,
|
|
106
|
+
source_name=source_name,
|
|
104
107
|
)
|
|
105
108
|
|
|
106
109
|
@classmethod
|
|
@@ -283,3 +286,7 @@ class InferenceImporter(BaseRDFImporter):
|
|
|
283
286
|
updated=now,
|
|
284
287
|
description="Inferred model from knowledge graph",
|
|
285
288
|
)
|
|
289
|
+
|
|
290
|
+
@property
|
|
291
|
+
def source_uri(self) -> URIRef:
|
|
292
|
+
return INSTANCES_ENTITY.id_
|
|
@@ -11,9 +11,9 @@ from typing import Literal, cast
|
|
|
11
11
|
import pandas as pd
|
|
12
12
|
from cognite.client.utils._importing import local_import
|
|
13
13
|
from pandas import ExcelFile
|
|
14
|
-
from rdflib import Namespace
|
|
14
|
+
from rdflib import Namespace, URIRef
|
|
15
15
|
|
|
16
|
-
from cognite.neat._issues import IssueList
|
|
16
|
+
from cognite.neat._issues import IssueList, MultiValueError
|
|
17
17
|
from cognite.neat._issues.errors import (
|
|
18
18
|
FileMissingRequiredFieldError,
|
|
19
19
|
FileNotFoundNeatError,
|
|
@@ -253,18 +253,19 @@ class ExcelImporter(BaseImporter[T_InputRules]):
|
|
|
253
253
|
def to_rules(self) -> ReadRules[T_InputRules]:
|
|
254
254
|
issue_list = IssueList(title=f"'{self.filepath.name}'")
|
|
255
255
|
if not self.filepath.exists():
|
|
256
|
-
|
|
257
|
-
return ReadRules(None, issue_list, {})
|
|
256
|
+
raise FileNotFoundNeatError(self.filepath)
|
|
258
257
|
|
|
259
258
|
with pd.ExcelFile(self.filepath) as excel_file:
|
|
260
259
|
user_reader = SpreadsheetReader(issue_list)
|
|
261
260
|
|
|
262
261
|
user_read = user_reader.read(excel_file, self.filepath)
|
|
263
|
-
if user_read is None or issue_list.has_errors:
|
|
264
|
-
return ReadRules(None, issue_list, {})
|
|
265
262
|
|
|
263
|
+
issue_list.trigger_warnings()
|
|
266
264
|
if issue_list.has_errors:
|
|
267
|
-
|
|
265
|
+
raise MultiValueError(issue_list.errors)
|
|
266
|
+
|
|
267
|
+
if user_read is None:
|
|
268
|
+
return ReadRules(None, {})
|
|
268
269
|
|
|
269
270
|
sheets = user_read.sheets
|
|
270
271
|
original_role = user_read.role
|
|
@@ -272,7 +273,15 @@ class ExcelImporter(BaseImporter[T_InputRules]):
|
|
|
272
273
|
|
|
273
274
|
rules_cls = INPUT_RULES_BY_ROLE[original_role]
|
|
274
275
|
rules = cast(T_InputRules, rules_cls.load(sheets))
|
|
275
|
-
return ReadRules(rules,
|
|
276
|
+
return ReadRules(rules, {"read_info_by_sheet": read_info_by_sheet})
|
|
277
|
+
|
|
278
|
+
@property
|
|
279
|
+
def description(self) -> str:
|
|
280
|
+
return f"Excel file {self.filepath.name} read as unverified data model"
|
|
281
|
+
|
|
282
|
+
@property
|
|
283
|
+
def source_uri(self) -> URIRef:
|
|
284
|
+
return URIRef(f"file://{self.filepath.name}")
|
|
276
285
|
|
|
277
286
|
|
|
278
287
|
class GoogleSheetImporter(BaseImporter[T_InputRules]):
|
|
@@ -3,7 +3,7 @@ from typing import Any, cast
|
|
|
3
3
|
|
|
4
4
|
import yaml
|
|
5
5
|
|
|
6
|
-
from cognite.neat._issues import IssueList, NeatIssue
|
|
6
|
+
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
|
|
7
7
|
from cognite.neat._issues.errors import (
|
|
8
8
|
FileMissingRequiredFieldError,
|
|
9
9
|
FileNotAFileError,
|
|
@@ -36,24 +36,32 @@ class YAMLImporter(BaseImporter[T_InputRules]):
|
|
|
36
36
|
raw_data: dict[str, Any],
|
|
37
37
|
read_issues: list[NeatIssue] | None = None,
|
|
38
38
|
filepaths: list[Path] | None = None,
|
|
39
|
+
source_name: str = "Unknown",
|
|
39
40
|
) -> None:
|
|
40
41
|
self.raw_data = raw_data
|
|
41
42
|
self._read_issues = IssueList(read_issues)
|
|
42
43
|
self._filepaths = filepaths
|
|
44
|
+
self._source_name = source_name
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def description(self) -> str:
|
|
48
|
+
return f"YAML file {self._source_name} read as unverified data model"
|
|
43
49
|
|
|
44
50
|
@classmethod
|
|
45
|
-
def from_file(cls, filepath: Path):
|
|
51
|
+
def from_file(cls, filepath: Path, source_name: str = "Unknown") -> "YAMLImporter":
|
|
46
52
|
if not filepath.exists():
|
|
47
53
|
return cls({}, [FileNotFoundNeatError(filepath)])
|
|
48
54
|
elif not filepath.is_file():
|
|
49
55
|
return cls({}, [FileNotAFileError(filepath)])
|
|
50
56
|
elif filepath.suffix not in [".yaml", ".yml"]:
|
|
51
57
|
return cls({}, [FileTypeUnexpectedError(filepath, frozenset([".yaml", ".yml"]))])
|
|
52
|
-
return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
|
|
58
|
+
return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath], source_name=source_name)
|
|
53
59
|
|
|
54
60
|
def to_rules(self) -> ReadRules[T_InputRules]:
|
|
55
61
|
if self._read_issues.has_errors or not self.raw_data:
|
|
56
|
-
|
|
62
|
+
self._read_issues.trigger_warnings()
|
|
63
|
+
raise MultiValueError(self._read_issues.errors)
|
|
64
|
+
|
|
57
65
|
issue_list = IssueList(title="YAML Importer", issues=self._read_issues)
|
|
58
66
|
|
|
59
67
|
if not self._filepaths:
|
|
@@ -69,13 +77,15 @@ class YAMLImporter(BaseImporter[T_InputRules]):
|
|
|
69
77
|
|
|
70
78
|
if "metadata" not in self.raw_data:
|
|
71
79
|
self._read_issues.append(FileMissingRequiredFieldError(metadata_file, "section", "metadata"))
|
|
72
|
-
|
|
80
|
+
issue_list.trigger_warnings()
|
|
81
|
+
raise MultiValueError(self._read_issues.errors)
|
|
73
82
|
|
|
74
83
|
metadata = self.raw_data["metadata"]
|
|
75
84
|
|
|
76
85
|
if "role" not in metadata:
|
|
77
86
|
self._read_issues.append(FileMissingRequiredFieldError(metadata, "metadata", "role"))
|
|
78
|
-
|
|
87
|
+
issue_list.trigger_warnings()
|
|
88
|
+
raise MultiValueError(self._read_issues.errors)
|
|
79
89
|
|
|
80
90
|
role_input = RoleTypes(metadata["role"])
|
|
81
91
|
role_enum = RoleTypes(role_input)
|
|
@@ -83,4 +93,8 @@ class YAMLImporter(BaseImporter[T_InputRules]):
|
|
|
83
93
|
|
|
84
94
|
rules = cast(T_InputRules, rules_cls.load(self.raw_data))
|
|
85
95
|
|
|
86
|
-
|
|
96
|
+
issue_list.trigger_warnings()
|
|
97
|
+
if self._read_issues.has_errors:
|
|
98
|
+
raise MultiValueError(self._read_issues.errors)
|
|
99
|
+
|
|
100
|
+
return ReadRules[T_InputRules](rules, {})
|
|
@@ -110,7 +110,7 @@ class InputRules(Generic[T_BaseRules], ABC):
|
|
|
110
110
|
def _dataclass_fields(self) -> list[Field]:
|
|
111
111
|
return list(fields(self))
|
|
112
112
|
|
|
113
|
-
def
|
|
113
|
+
def as_verified_rules(self) -> T_BaseRules:
|
|
114
114
|
cls_ = self._get_verified_cls()
|
|
115
115
|
return cls_.model_validate(self.dump())
|
|
116
116
|
|
|
@@ -50,6 +50,7 @@ from cognite.neat._rules.models._types import (
|
|
|
50
50
|
)
|
|
51
51
|
from cognite.neat._rules.models.data_types import DataType
|
|
52
52
|
from cognite.neat._rules.models.entities import EdgeEntity, ReverseConnectionEntity, ViewEntity
|
|
53
|
+
from cognite.neat._utils.rdf_ import uri_display_name
|
|
53
54
|
|
|
54
55
|
if sys.version_info >= (3, 11):
|
|
55
56
|
from enum import StrEnum
|
|
@@ -285,6 +286,10 @@ class BaseRules(SchemaModel, ABC):
|
|
|
285
286
|
]
|
|
286
287
|
return headers_by_sheet
|
|
287
288
|
|
|
289
|
+
@property
|
|
290
|
+
def display_name(self):
|
|
291
|
+
return uri_display_name(self.metadata.identifier)
|
|
292
|
+
|
|
288
293
|
def dump(
|
|
289
294
|
self,
|
|
290
295
|
entities_exclude_defaults: bool = True,
|
|
@@ -447,6 +447,10 @@ class DMSRules(BaseRules):
|
|
|
447
447
|
|
|
448
448
|
return _DMSExporter(self, instance_space, remove_cdf_spaces=remove_cdf_spaces).to_schema()
|
|
449
449
|
|
|
450
|
+
@classmethod
|
|
451
|
+
def display_type_name(cls) -> str:
|
|
452
|
+
return "VerifiedDMSModel"
|
|
453
|
+
|
|
450
454
|
def _repr_html_(self) -> str:
|
|
451
455
|
summary = {
|
|
452
456
|
"aspect": self.metadata.aspect,
|
|
@@ -21,6 +21,7 @@ from cognite.neat._rules.models.entities import (
|
|
|
21
21
|
load_connection,
|
|
22
22
|
load_dms_value_type,
|
|
23
23
|
)
|
|
24
|
+
from cognite.neat._utils.rdf_ import uri_display_name
|
|
24
25
|
|
|
25
26
|
from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
|
|
26
27
|
|
|
@@ -289,6 +290,14 @@ class DMSInputRules(InputRules[DMSRules]):
|
|
|
289
290
|
"Nodes": [node_type.dump(default_space) for node_type in self.nodes or []] or None,
|
|
290
291
|
}
|
|
291
292
|
|
|
293
|
+
@classmethod
|
|
294
|
+
def display_type_name(cls) -> str:
|
|
295
|
+
return "UnverifiedDMSModel"
|
|
296
|
+
|
|
297
|
+
@property
|
|
298
|
+
def display_name(self):
|
|
299
|
+
return uri_display_name(self.metadata.identifier)
|
|
300
|
+
|
|
292
301
|
def _repr_html_(self) -> str:
|
|
293
302
|
summary = {
|
|
294
303
|
"type": "Physical Data Model",
|
|
@@ -194,6 +194,8 @@ class DMSValidation:
|
|
|
194
194
|
) -> dict[ViewId, set[ViewId]]:
|
|
195
195
|
@lru_cache
|
|
196
196
|
def get_parents(child_view_id: ViewId) -> set[ViewId]:
|
|
197
|
+
if child_view_id not in all_views_by_id:
|
|
198
|
+
return set()
|
|
197
199
|
child_view = all_views_by_id[child_view_id]
|
|
198
200
|
parents = set(child_view.implements or [])
|
|
199
201
|
for parent_id in child_view.implements or []:
|
|
@@ -201,16 +201,36 @@ class Entity(BaseModel, extra="ignore"):
|
|
|
201
201
|
for field_name, field in self.model_fields.items()
|
|
202
202
|
if (v := getattr(self, field_name)) is not None and field_name not in {"prefix", "suffix"}
|
|
203
203
|
}
|
|
204
|
+
# We only remove the default values if all the fields are default
|
|
205
|
+
# For example, if we dump `cdf_cdm:CogniteAsset(version=v1)` and the default is `version=v1`,
|
|
206
|
+
# we should not remove it unless the space is `cdf_cdm`
|
|
207
|
+
to_delete: list[str] = []
|
|
208
|
+
is_removing_defaults = True
|
|
204
209
|
if isinstance(defaults, dict):
|
|
205
210
|
for key, value in defaults.items():
|
|
206
|
-
if key in model_dump
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
+
if key not in model_dump:
|
|
212
|
+
continue
|
|
213
|
+
if model_dump[key] == value:
|
|
214
|
+
to_delete.append(key)
|
|
215
|
+
elif isinstance(self, EdgeEntity):
|
|
216
|
+
# Exception is edge entity, then, we remove all the defaults we can.
|
|
217
|
+
continue
|
|
218
|
+
else:
|
|
219
|
+
# Not all fields are default. We should not remove any of them.
|
|
220
|
+
is_removing_defaults = False
|
|
221
|
+
break
|
|
222
|
+
if isinstance(defaults, dict) and self.prefix == defaults.get("prefix") and is_removing_defaults:
|
|
223
|
+
base_id = str(self.suffix)
|
|
224
|
+
elif self.prefix == Undefined:
|
|
211
225
|
base_id = str(self.suffix)
|
|
212
226
|
else:
|
|
227
|
+
is_removing_defaults = False
|
|
213
228
|
base_id = f"{self.prefix}:{self.suffix!s}"
|
|
229
|
+
if is_removing_defaults:
|
|
230
|
+
for key in to_delete:
|
|
231
|
+
del model_dump[key]
|
|
232
|
+
# Sorting to ensure deterministic order
|
|
233
|
+
args = ",".join(f"{k}={v}" for k, v in sorted(model_dump.items(), key=lambda x: x[0]))
|
|
214
234
|
if args:
|
|
215
235
|
return f"{base_id}({args})"
|
|
216
236
|
else:
|
|
@@ -272,6 +272,10 @@ class InformationRules(BaseRules):
|
|
|
272
272
|
|
|
273
273
|
return _InformationRulesConverter(self).as_dms_rules()
|
|
274
274
|
|
|
275
|
+
@classmethod
|
|
276
|
+
def display_type_name(cls) -> str:
|
|
277
|
+
return "VerifiedInformationModel"
|
|
278
|
+
|
|
275
279
|
def _repr_html_(self) -> str:
|
|
276
280
|
summary = {
|
|
277
281
|
"type": "Logical Data Model",
|
|
@@ -14,6 +14,7 @@ from cognite.neat._rules.models.entities import (
|
|
|
14
14
|
UnknownEntity,
|
|
15
15
|
load_value_type,
|
|
16
16
|
)
|
|
17
|
+
from cognite.neat._utils.rdf_ import uri_display_name
|
|
17
18
|
|
|
18
19
|
from ._rules import (
|
|
19
20
|
InformationClass,
|
|
@@ -150,6 +151,14 @@ class InformationInputRules(InputRules[InformationRules]):
|
|
|
150
151
|
Prefixes=self.prefixes,
|
|
151
152
|
)
|
|
152
153
|
|
|
154
|
+
@classmethod
|
|
155
|
+
def display_type_name(cls) -> str:
|
|
156
|
+
return "UnverifiedInformationModel"
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def display_name(self):
|
|
160
|
+
return uri_display_name(self.metadata.identifier)
|
|
161
|
+
|
|
153
162
|
def _repr_html_(self) -> str:
|
|
154
163
|
summary = {
|
|
155
164
|
"type": "Logical Data Model",
|
|
@@ -32,9 +32,6 @@ def load_classic_to_core_mapping(org_name: str, source_space: str, source_versio
|
|
|
32
32
|
if not isinstance(read.rules, DMSInputRules):
|
|
33
33
|
raise NeatValueError(f"Expected DMS rules, but got {type(read.rules).__name__}")
|
|
34
34
|
|
|
35
|
-
verified = VerifyDMSRules(
|
|
35
|
+
verified = VerifyDMSRules(validate=False).transform(read)
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
raise NeatValueError("Failed to verify the rules.")
|
|
39
|
-
|
|
40
|
-
return verified.rules
|
|
37
|
+
return verified
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
from ._base import
|
|
1
|
+
from ._base import RulesTransformer
|
|
2
2
|
from ._converters import (
|
|
3
|
+
AddClassImplements,
|
|
3
4
|
ConvertToRules,
|
|
4
5
|
DMSToInformation,
|
|
6
|
+
IncludeReferenced,
|
|
5
7
|
InformationToDMS,
|
|
6
8
|
PrefixEntities,
|
|
7
9
|
ReduceCogniteModel,
|
|
@@ -10,20 +12,19 @@ from ._converters import (
|
|
|
10
12
|
ToExtension,
|
|
11
13
|
)
|
|
12
14
|
from ._mapping import AsParentPropertyId, MapOneToOne, RuleMapper
|
|
13
|
-
from ._pipelines import ImporterPipeline
|
|
14
15
|
from ._verification import VerifyAnyRules, VerifyDMSRules, VerifyInformationRules
|
|
15
16
|
|
|
16
17
|
__all__ = [
|
|
18
|
+
"AddClassImplements",
|
|
17
19
|
"AsParentPropertyId",
|
|
18
20
|
"ConvertToRules",
|
|
19
21
|
"DMSToInformation",
|
|
20
|
-
"
|
|
22
|
+
"IncludeReferenced",
|
|
21
23
|
"InformationToDMS",
|
|
22
24
|
"MapOneToOne",
|
|
23
25
|
"PrefixEntities",
|
|
24
26
|
"ReduceCogniteModel",
|
|
25
27
|
"RuleMapper",
|
|
26
|
-
"RulesPipeline",
|
|
27
28
|
"RulesTransformer",
|
|
28
29
|
"SetIDDMSModel",
|
|
29
30
|
"ToCompliantEntities",
|
|
@@ -1,18 +1,12 @@
|
|
|
1
|
+
import inspect
|
|
1
2
|
from abc import ABC, abstractmethod
|
|
2
|
-
from
|
|
3
|
-
from
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
from types import UnionType
|
|
5
|
+
from typing import Generic, TypeVar, Union, get_args, get_origin
|
|
4
6
|
|
|
5
7
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
6
|
-
from cognite.neat.
|
|
7
|
-
from cognite.neat.
|
|
8
|
-
from cognite.neat._rules._shared import (
|
|
9
|
-
InputRules,
|
|
10
|
-
JustRules,
|
|
11
|
-
MaybeRules,
|
|
12
|
-
OutRules,
|
|
13
|
-
Rules,
|
|
14
|
-
VerifiedRules,
|
|
15
|
-
)
|
|
8
|
+
from cognite.neat._rules._shared import ReadRules, Rules
|
|
9
|
+
from cognite.neat._rules.models import DMSInputRules, InformationInputRules
|
|
16
10
|
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
17
11
|
|
|
18
12
|
T_RulesIn = TypeVar("T_RulesIn", bound=Rules)
|
|
@@ -20,69 +14,51 @@ T_RulesOut = TypeVar("T_RulesOut", bound=Rules)
|
|
|
20
14
|
|
|
21
15
|
|
|
22
16
|
class RulesTransformer(ABC, Generic[T_RulesIn, T_RulesOut]):
|
|
23
|
-
"""This is the base class for all rule transformers.
|
|
24
|
-
|
|
25
|
-
Note transformers follow the functional pattern Monad
|
|
26
|
-
https://en.wikipedia.org/wiki/Monad_(functional_programming)
|
|
27
|
-
"""
|
|
17
|
+
"""This is the base class for all rule transformers."""
|
|
28
18
|
|
|
29
19
|
@abstractmethod
|
|
30
|
-
def transform(self, rules: T_RulesIn
|
|
20
|
+
def transform(self, rules: T_RulesIn) -> T_RulesOut:
|
|
31
21
|
"""Transform the input rules into the output rules."""
|
|
32
22
|
raise NotImplementedError()
|
|
33
23
|
|
|
34
|
-
def try_transform(self, rules: MaybeRules[T_RulesIn]) -> MaybeRules[T_RulesOut]:
|
|
35
|
-
"""Try to transform the input rules into the output rules."""
|
|
36
|
-
try:
|
|
37
|
-
result = self.transform(rules)
|
|
38
|
-
except NeatError:
|
|
39
|
-
# Any error caught during transformation will be returned as issues
|
|
40
|
-
return MaybeRules(None, rules.issues)
|
|
41
|
-
issues = IssueList(rules.issues, title=rules.issues.title)
|
|
42
|
-
if isinstance(result, MaybeRules):
|
|
43
|
-
issues.extend(result.issues)
|
|
44
|
-
return MaybeRules(result.get_rules(), issues)
|
|
45
|
-
|
|
46
|
-
@classmethod
|
|
47
|
-
def _to_rules(cls, rules: T_RulesIn | OutRules[T_RulesIn]) -> T_RulesIn:
|
|
48
|
-
if isinstance(rules, JustRules):
|
|
49
|
-
return rules.rules
|
|
50
|
-
elif isinstance(rules, MaybeRules):
|
|
51
|
-
if rules.rules is None:
|
|
52
|
-
raise NeatValueError("Rules is missing cannot convert")
|
|
53
|
-
return rules.rules
|
|
54
|
-
elif isinstance(rules, VerifiedRules | InputRules):
|
|
55
|
-
return rules # type: ignore[return-value]
|
|
56
|
-
else:
|
|
57
|
-
raise NeatTypeError(f"Unsupported type: {type(rules)}")
|
|
58
|
-
|
|
59
24
|
@property
|
|
60
25
|
def agent(self) -> ProvenanceAgent:
|
|
61
26
|
"""Provenance agent for the importer."""
|
|
62
27
|
return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
|
|
63
28
|
|
|
29
|
+
@property
|
|
30
|
+
def description(self) -> str:
|
|
31
|
+
"""Get the description of the transformer."""
|
|
32
|
+
return "MISSING DESCRIPTION"
|
|
64
33
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
for
|
|
69
|
-
|
|
70
|
-
|
|
34
|
+
def is_valid_input(self, rules: T_RulesIn) -> bool:
|
|
35
|
+
"""Check if the input rules are valid."""
|
|
36
|
+
types = self.transform_type_hint()
|
|
37
|
+
for type_ in types:
|
|
38
|
+
if get_origin(type_) is ReadRules:
|
|
39
|
+
inner = get_args(type_)[0]
|
|
40
|
+
if isinstance(rules, ReadRules) and isinstance(rules.rules, inner):
|
|
41
|
+
return True
|
|
42
|
+
elif isinstance(rules, type_):
|
|
43
|
+
return True
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
@lru_cache(maxsize=1)
|
|
48
|
+
def transform_type_hint(cls) -> tuple[type, ...]:
|
|
49
|
+
# This is an expensive operation, so we cache the result
|
|
50
|
+
signature = inspect.signature(cls.transform)
|
|
51
|
+
annotation = signature.parameters["rules"].annotation
|
|
52
|
+
if isinstance(annotation, TypeVar):
|
|
53
|
+
if annotation.__bound__ is None:
|
|
54
|
+
raise TypeError(f"TypeVar {annotation} must be bound to a type.")
|
|
55
|
+
annotation = annotation.__bound__
|
|
56
|
+
# The annotation can be a type or a generic
|
|
57
|
+
if get_origin(annotation) in [UnionType, Union]:
|
|
58
|
+
return get_args(annotation)
|
|
71
59
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
rules = transformer.try_transform(rules)
|
|
76
|
-
return rules # type: ignore[return-value]
|
|
60
|
+
if get_origin(annotation) is ReadRules and isinstance(get_args(annotation)[0], TypeVar):
|
|
61
|
+
# Hardcoded for now, as we only have two types of ReadRules
|
|
62
|
+
return ReadRules[DMSInputRules], ReadRules[InformationInputRules]
|
|
77
63
|
|
|
78
|
-
|
|
79
|
-
"""Run the pipeline from the input rules to the output rules."""
|
|
80
|
-
output = self.transform(rules)
|
|
81
|
-
if isinstance(output, MaybeRules):
|
|
82
|
-
if output.rules is None:
|
|
83
|
-
raise NeatValueError(f"Rule transformation failed: {output.issues}")
|
|
84
|
-
return output.rules
|
|
85
|
-
elif isinstance(output, JustRules):
|
|
86
|
-
return output.rules
|
|
87
|
-
else:
|
|
88
|
-
raise NeatTypeError(f"Rule transformation failed: {output}")
|
|
64
|
+
return (annotation,)
|