cognite-neat 0.88.2__py3-none-any.whl → 0.88.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_version.py +1 -1
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/loaders/_base.py +3 -3
- cognite/neat/graph/loaders/_rdf2asset.py +24 -25
- cognite/neat/graph/loaders/_rdf2dms.py +20 -15
- cognite/neat/issues/__init__.py +1 -3
- cognite/neat/issues/_base.py +259 -70
- cognite/neat/issues/errors/__init__.py +72 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +28 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/issues/formatters.py +1 -1
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +10 -2
- cognite/neat/rules/exporters/_base.py +6 -6
- cognite/neat/rules/exporters/_rules2dms.py +18 -11
- cognite/neat/rules/exporters/_rules2excel.py +4 -4
- cognite/neat/rules/exporters/_rules2ontology.py +74 -51
- cognite/neat/rules/exporters/_rules2yaml.py +3 -3
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/_base.py +8 -12
- cognite/neat/rules/importers/_dms2rules.py +21 -24
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +22 -17
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +26 -19
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +1 -1
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +9 -7
- cognite/neat/rules/importers/_rdf/_inference2rules.py +8 -8
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +4 -4
- cognite/neat/rules/importers/_rdf/_shared.py +3 -3
- cognite/neat/rules/importers/_spreadsheet2rules.py +35 -22
- cognite/neat/rules/importers/_yaml2rules.py +23 -22
- cognite/neat/rules/models/_constants.py +2 -1
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +5 -10
- cognite/neat/rules/models/asset/_rules.py +1 -3
- cognite/neat/rules/models/asset/_validation.py +13 -9
- cognite/neat/rules/models/dms/_converter.py +2 -4
- cognite/neat/rules/models/dms/_exporter.py +30 -8
- cognite/neat/rules/models/dms/_rules.py +23 -7
- cognite/neat/rules/models/dms/_schema.py +87 -78
- cognite/neat/rules/models/dms/_validation.py +104 -65
- cognite/neat/rules/models/information/_converter.py +2 -2
- cognite/neat/rules/models/information/_rules.py +7 -8
- cognite/neat/rules/models/information/_validation.py +47 -24
- cognite/neat/rules/transformers/_base.py +15 -0
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -10
- cognite/neat/workflows/steps/lib/current/rules_importer.py +6 -6
- cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/RECORD +78 -84
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/issues/errors/external.py +0 -21
- cognite/neat/issues/errors/properties.py +0 -75
- cognite/neat/issues/errors/resources.py +0 -123
- cognite/neat/issues/neat_warnings/__init__.py +0 -2
- cognite/neat/issues/neat_warnings/identifier.py +0 -27
- cognite/neat/issues/neat_warnings/models.py +0 -22
- cognite/neat/issues/neat_warnings/properties.py +0 -77
- cognite/neat/issues/neat_warnings/resources.py +0 -125
- cognite/neat/rules/issues/__init__.py +0 -22
- cognite/neat/rules/issues/base.py +0 -63
- cognite/neat/rules/issues/dms.py +0 -549
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{issues/errors/schema.py → rules/transformers/__init__.py} +0 -0
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- /cognite/neat/{graph/stores → store}/_provenance.py +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/entry_points.txt +0 -0
|
@@ -11,8 +11,13 @@ from typing import Literal, cast, overload
|
|
|
11
11
|
import pandas as pd
|
|
12
12
|
from pandas import ExcelFile
|
|
13
13
|
|
|
14
|
-
from cognite.neat.issues import IssueList
|
|
15
|
-
from cognite.neat.
|
|
14
|
+
from cognite.neat.issues import IssueList, NeatError
|
|
15
|
+
from cognite.neat.issues.errors import (
|
|
16
|
+
FileMissingRequiredFieldError,
|
|
17
|
+
FileNotFoundNeatError,
|
|
18
|
+
FileReadError,
|
|
19
|
+
PropertyDefinitionDuplicatedError,
|
|
20
|
+
)
|
|
16
21
|
from cognite.neat.rules.models import (
|
|
17
22
|
RULES_PER_ROLE,
|
|
18
23
|
AssetRules,
|
|
@@ -27,8 +32,9 @@ from cognite.neat.rules.models.dms import DMSRulesInput
|
|
|
27
32
|
from cognite.neat.rules.models.information import InformationRulesInput
|
|
28
33
|
from cognite.neat.utils.auxiliary import local_import
|
|
29
34
|
from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
|
|
35
|
+
from cognite.neat.utils.text import humanize_collection
|
|
30
36
|
|
|
31
|
-
from ._base import BaseImporter,
|
|
37
|
+
from ._base import BaseImporter, VerifiedRules, _handle_issues
|
|
32
38
|
|
|
33
39
|
SOURCE_SHEET__TARGET_FIELD__HEADERS = [
|
|
34
40
|
(
|
|
@@ -77,12 +83,12 @@ class MetadataRaw(UserDict):
|
|
|
77
83
|
|
|
78
84
|
def is_valid(self, issue_list: IssueList, filepath: Path) -> bool:
|
|
79
85
|
if not self.has_role_field:
|
|
80
|
-
issue_list.append(
|
|
86
|
+
issue_list.append(FileMissingRequiredFieldError(filepath, "metadata", "role"))
|
|
81
87
|
return False
|
|
82
88
|
|
|
83
89
|
# check if there is a schema field if role is not domain expert
|
|
84
90
|
if self.role != RoleTypes.domain_expert and not self.has_schema_field:
|
|
85
|
-
issue_list.append(
|
|
91
|
+
issue_list.append(FileMissingRequiredFieldError(filepath, "metadata", "schema"))
|
|
86
92
|
return False
|
|
87
93
|
return True
|
|
88
94
|
|
|
@@ -153,11 +159,7 @@ class SpreadsheetReader:
|
|
|
153
159
|
def _read_metadata(self, excel_file: ExcelFile, filepath: Path) -> MetadataRaw | None:
|
|
154
160
|
if self.metadata_sheet_name not in excel_file.sheet_names:
|
|
155
161
|
if self.required:
|
|
156
|
-
self.issue_list.append(
|
|
157
|
-
issues.spreadsheet_file.MetadataSheetMissingOrFailedError(
|
|
158
|
-
filepath, sheet_name=self.metadata_sheet_name
|
|
159
|
-
)
|
|
160
|
-
)
|
|
162
|
+
self.issue_list.append(FileMissingRequiredFieldError(filepath, "sheet", self.metadata_sheet_name))
|
|
161
163
|
return None
|
|
162
164
|
|
|
163
165
|
metadata = MetadataRaw.from_excel(excel_file, self.metadata_sheet_name)
|
|
@@ -178,7 +180,9 @@ class SpreadsheetReader:
|
|
|
178
180
|
if missing_sheets := expected_sheet_names.difference(set(excel_file.sheet_names)):
|
|
179
181
|
if self.required:
|
|
180
182
|
self.issue_list.append(
|
|
181
|
-
|
|
183
|
+
FileMissingRequiredFieldError(
|
|
184
|
+
cast(Path, excel_file.io), "sheets", humanize_collection(missing_sheets)
|
|
185
|
+
)
|
|
182
186
|
)
|
|
183
187
|
return None, read_info_by_sheet
|
|
184
188
|
|
|
@@ -197,7 +201,7 @@ class SpreadsheetReader:
|
|
|
197
201
|
excel_file, source_sheet_name, return_read_info=True, expected_headers=[headers]
|
|
198
202
|
)
|
|
199
203
|
except Exception as e:
|
|
200
|
-
self.issue_list.append(
|
|
204
|
+
self.issue_list.append(FileReadError(cast(Path, excel_file.io), str(e)))
|
|
201
205
|
continue
|
|
202
206
|
|
|
203
207
|
return sheets, read_info_by_sheet
|
|
@@ -214,21 +218,21 @@ class ExcelImporter(BaseImporter):
|
|
|
214
218
|
self.filepath = filepath
|
|
215
219
|
|
|
216
220
|
@overload
|
|
217
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
221
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
218
222
|
|
|
219
223
|
@overload
|
|
220
224
|
def to_rules(
|
|
221
225
|
self,
|
|
222
226
|
errors: Literal["continue"] = "continue",
|
|
223
227
|
role: RoleTypes | None = None,
|
|
224
|
-
) -> tuple[
|
|
228
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
225
229
|
|
|
226
230
|
def to_rules(
|
|
227
231
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
228
|
-
) -> tuple[
|
|
232
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
229
233
|
issue_list = IssueList(title=f"'{self.filepath.name}'")
|
|
230
234
|
if not self.filepath.exists():
|
|
231
|
-
issue_list.append(
|
|
235
|
+
issue_list.append(FileNotFoundNeatError(self.filepath))
|
|
232
236
|
return self._return_or_raise(issue_list, errors)
|
|
233
237
|
|
|
234
238
|
with pd.ExcelFile(self.filepath) as excel_file:
|
|
@@ -251,7 +255,16 @@ class ExcelImporter(BaseImporter):
|
|
|
251
255
|
return self._return_or_raise(issue_list, errors)
|
|
252
256
|
|
|
253
257
|
if reference_read and user_read.role != reference_read.role:
|
|
254
|
-
issue_list.append(
|
|
258
|
+
issue_list.append(
|
|
259
|
+
PropertyDefinitionDuplicatedError(
|
|
260
|
+
self.filepath.as_posix(),
|
|
261
|
+
"spreadsheet.metadata", # type: ignore[arg-type]
|
|
262
|
+
"role",
|
|
263
|
+
frozenset({user_read.role, reference_read.role}),
|
|
264
|
+
("user", "reference"),
|
|
265
|
+
"sheet",
|
|
266
|
+
)
|
|
267
|
+
)
|
|
255
268
|
return self._return_or_raise(issue_list, errors)
|
|
256
269
|
|
|
257
270
|
sheets = user_read.sheets
|
|
@@ -270,10 +283,10 @@ class ExcelImporter(BaseImporter):
|
|
|
270
283
|
rules_cls = RULES_PER_ROLE[original_role]
|
|
271
284
|
with _handle_issues(
|
|
272
285
|
issue_list,
|
|
273
|
-
error_cls=
|
|
286
|
+
error_cls=NeatError,
|
|
274
287
|
error_args={"read_info_by_sheet": read_info_by_sheet},
|
|
275
288
|
) as future:
|
|
276
|
-
rules:
|
|
289
|
+
rules: VerifiedRules
|
|
277
290
|
if rules_cls is DMSRules:
|
|
278
291
|
rules = DMSRulesInput.load(sheets).as_rules()
|
|
279
292
|
elif rules_cls is InformationRules:
|
|
@@ -311,16 +324,16 @@ class GoogleSheetImporter(BaseImporter):
|
|
|
311
324
|
self.skiprows = skiprows
|
|
312
325
|
|
|
313
326
|
@overload
|
|
314
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
327
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
315
328
|
|
|
316
329
|
@overload
|
|
317
330
|
def to_rules(
|
|
318
331
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
319
|
-
) -> tuple[
|
|
332
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
320
333
|
|
|
321
334
|
def to_rules(
|
|
322
335
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
323
|
-
) -> tuple[
|
|
336
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
324
337
|
local_import("gspread", "google")
|
|
325
338
|
import gspread # type: ignore[import]
|
|
326
339
|
|
|
@@ -3,13 +3,18 @@ from typing import Any, Literal, overload
|
|
|
3
3
|
|
|
4
4
|
import yaml
|
|
5
5
|
|
|
6
|
-
from cognite.neat.issues import IssueList
|
|
7
|
-
from cognite.neat.
|
|
8
|
-
|
|
6
|
+
from cognite.neat.issues import IssueList, NeatIssue
|
|
7
|
+
from cognite.neat.issues.errors import (
|
|
8
|
+
FileMissingRequiredFieldError,
|
|
9
|
+
FileNotAFileError,
|
|
10
|
+
FileNotFoundNeatError,
|
|
11
|
+
FileTypeUnexpectedError,
|
|
12
|
+
)
|
|
13
|
+
from cognite.neat.issues.warnings import NeatValueWarning
|
|
9
14
|
from cognite.neat.rules.models import RULES_PER_ROLE, DMSRules, RoleTypes
|
|
10
15
|
from cognite.neat.rules.models.dms import DMSRulesInput
|
|
11
16
|
|
|
12
|
-
from ._base import BaseImporter,
|
|
17
|
+
from ._base import BaseImporter, VerifiedRules, _handle_issues
|
|
13
18
|
|
|
14
19
|
|
|
15
20
|
class YAMLImporter(BaseImporter):
|
|
@@ -29,7 +34,7 @@ class YAMLImporter(BaseImporter):
|
|
|
29
34
|
def __init__(
|
|
30
35
|
self,
|
|
31
36
|
raw_data: dict[str, Any],
|
|
32
|
-
read_issues: list[
|
|
37
|
+
read_issues: list[NeatIssue] | None = None,
|
|
33
38
|
filepaths: list[Path] | None = None,
|
|
34
39
|
) -> None:
|
|
35
40
|
self.raw_data = raw_data
|
|
@@ -39,25 +44,25 @@ class YAMLImporter(BaseImporter):
|
|
|
39
44
|
@classmethod
|
|
40
45
|
def from_file(cls, filepath: Path):
|
|
41
46
|
if not filepath.exists():
|
|
42
|
-
return cls({}, [
|
|
43
|
-
|
|
44
|
-
return cls({}, [
|
|
47
|
+
return cls({}, [FileNotFoundNeatError(filepath)])
|
|
48
|
+
elif not filepath.is_file():
|
|
49
|
+
return cls({}, [FileNotAFileError(filepath)])
|
|
45
50
|
elif filepath.suffix not in [".yaml", ".yml"]:
|
|
46
|
-
return cls({}, [
|
|
51
|
+
return cls({}, [FileTypeUnexpectedError(filepath, frozenset([".yaml", ".yml"]))])
|
|
47
52
|
return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
|
|
48
53
|
|
|
49
54
|
@overload
|
|
50
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
55
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
51
56
|
|
|
52
57
|
@overload
|
|
53
58
|
def to_rules(
|
|
54
59
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
55
|
-
) -> tuple[
|
|
60
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
56
61
|
|
|
57
62
|
def to_rules(
|
|
58
63
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
59
|
-
) -> tuple[
|
|
60
|
-
if
|
|
64
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
65
|
+
if self._read_issues.has_errors or not self.raw_data:
|
|
61
66
|
if errors == "raise":
|
|
62
67
|
raise self._read_issues.as_errors()
|
|
63
68
|
return None, self._read_issues
|
|
@@ -65,8 +70,8 @@ class YAMLImporter(BaseImporter):
|
|
|
65
70
|
|
|
66
71
|
if not self._filepaths:
|
|
67
72
|
issue_list.append(
|
|
68
|
-
|
|
69
|
-
|
|
73
|
+
NeatValueWarning(
|
|
74
|
+
f"{type(self).__name__} was called without filepaths when there is content",
|
|
70
75
|
)
|
|
71
76
|
)
|
|
72
77
|
metadata_file = Path()
|
|
@@ -75,9 +80,7 @@ class YAMLImporter(BaseImporter):
|
|
|
75
80
|
metadata_file = metadata_file_nullable or self._filepaths[0]
|
|
76
81
|
|
|
77
82
|
if "metadata" not in self.raw_data:
|
|
78
|
-
self._read_issues.append(
|
|
79
|
-
issues.spreadsheet_file.MetadataSheetMissingOrFailedError(metadata_file, "Metadata not found in file")
|
|
80
|
-
)
|
|
83
|
+
self._read_issues.append(FileMissingRequiredFieldError(metadata_file, "section", "metadata"))
|
|
81
84
|
if errors == "raise":
|
|
82
85
|
raise self._read_issues.as_errors()
|
|
83
86
|
return None, self._read_issues
|
|
@@ -85,9 +88,7 @@ class YAMLImporter(BaseImporter):
|
|
|
85
88
|
metadata = self.raw_data["metadata"]
|
|
86
89
|
|
|
87
90
|
if "role" not in metadata:
|
|
88
|
-
self._read_issues.append(
|
|
89
|
-
issues.spreadsheet_file.MetadataSheetMissingOrFailedError(metadata_file, "Role not found in metadata")
|
|
90
|
-
)
|
|
91
|
+
self._read_issues.append(FileMissingRequiredFieldError(metadata, "metadata", "role"))
|
|
91
92
|
if errors == "raise":
|
|
92
93
|
raise self._read_issues.as_errors()
|
|
93
94
|
return None, self._read_issues
|
|
@@ -97,7 +98,7 @@ class YAMLImporter(BaseImporter):
|
|
|
97
98
|
rules_model = RULES_PER_ROLE[role_enum]
|
|
98
99
|
|
|
99
100
|
with _handle_issues(issue_list) as future:
|
|
100
|
-
rules:
|
|
101
|
+
rules: VerifiedRules
|
|
101
102
|
if rules_model is DMSRules:
|
|
102
103
|
rules = DMSRulesInput.load(self.raw_data).as_rules()
|
|
103
104
|
else:
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
DMS_CONTAINER_PROPERTY_SIZE_LIMIT = 100
|
|
2
|
+
DMS_VIEW_CONTAINER_SIZE_LIMIT = 10
|
|
@@ -8,7 +8,7 @@ from typing import ClassVar, Literal
|
|
|
8
8
|
|
|
9
9
|
from pydantic import BaseModel, field_validator, model_serializer
|
|
10
10
|
|
|
11
|
-
from cognite.neat.
|
|
11
|
+
from cognite.neat.issues.errors import NeatValueError
|
|
12
12
|
|
|
13
13
|
if sys.version_info >= (3, 11):
|
|
14
14
|
from enum import StrEnum
|
|
@@ -313,7 +313,7 @@ def parse_traversal(raw: str) -> SelfReferenceProperty | SingleProperty | Hop:
|
|
|
313
313
|
elif result := HOP_REGEX_COMPILED.match(raw):
|
|
314
314
|
return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
|
|
315
315
|
else:
|
|
316
|
-
raise
|
|
316
|
+
raise NeatValueError(f"Invalid RDF Path: {raw!r}")
|
|
317
317
|
|
|
318
318
|
|
|
319
319
|
def parse_table_lookup(raw: str) -> TableLookup:
|
|
@@ -323,7 +323,7 @@ def parse_table_lookup(raw: str) -> TableLookup:
|
|
|
323
323
|
key=result.group(Lookup.key),
|
|
324
324
|
value=result.group(Lookup.value),
|
|
325
325
|
)
|
|
326
|
-
raise
|
|
326
|
+
raise NeatValueError(f"Invalid table lookup: {raw!r}")
|
|
327
327
|
|
|
328
328
|
|
|
329
329
|
def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPath:
|
|
@@ -334,7 +334,7 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
|
|
|
334
334
|
case TransformationRuleType.rawlookup:
|
|
335
335
|
rule_raw = rule_raw.replace(" ", "")
|
|
336
336
|
if Counter(rule_raw).get("|") != 1:
|
|
337
|
-
raise
|
|
337
|
+
raise NeatValueError(f"Invalid rawlookup rule: {rule_raw!r}")
|
|
338
338
|
traversal, table_lookup = rule_raw.split("|")
|
|
339
339
|
return RawLookup(
|
|
340
340
|
traversal=parse_traversal(traversal),
|
|
@@ -14,10 +14,9 @@ from pydantic import (
|
|
|
14
14
|
WrapValidator,
|
|
15
15
|
)
|
|
16
16
|
from pydantic.functional_serializers import PlainSerializer
|
|
17
|
-
from pydantic_core import PydanticCustomError
|
|
18
17
|
|
|
19
|
-
from cognite.neat.issues.
|
|
20
|
-
from cognite.neat.
|
|
18
|
+
from cognite.neat.issues.errors import RegexViolationError
|
|
19
|
+
from cognite.neat.issues.warnings import RegexViolationWarning
|
|
21
20
|
from cognite.neat.utils.regex_patterns import (
|
|
22
21
|
PATTERNS,
|
|
23
22
|
PREFIX_COMPLIANCE_REGEX,
|
|
@@ -36,10 +35,6 @@ def _custom_error(exc_factory: Callable[[str | None, Exception], Any]) -> Any:
|
|
|
36
35
|
return WrapValidator(_validator)
|
|
37
36
|
|
|
38
37
|
|
|
39
|
-
def _raise(exception: PydanticCustomError):
|
|
40
|
-
raise exception
|
|
41
|
-
|
|
42
|
-
|
|
43
38
|
StrOrListType = Annotated[
|
|
44
39
|
str | list[str],
|
|
45
40
|
BeforeValidator(lambda value: value.replace(", ", ",").split(",") if isinstance(value, str) and value else value),
|
|
@@ -72,7 +67,7 @@ NamespaceType = Annotated[
|
|
|
72
67
|
PrefixType = Annotated[
|
|
73
68
|
str,
|
|
74
69
|
StringConstraints(pattern=PREFIX_COMPLIANCE_REGEX),
|
|
75
|
-
_custom_error(lambda _, value: RegexViolationError(value, PREFIX_COMPLIANCE_REGEX)
|
|
70
|
+
_custom_error(lambda _, value: RegexViolationError(value, PREFIX_COMPLIANCE_REGEX)),
|
|
76
71
|
]
|
|
77
72
|
|
|
78
73
|
ExternalIdType = Annotated[
|
|
@@ -83,13 +78,13 @@ ExternalIdType = Annotated[
|
|
|
83
78
|
VersionType = Annotated[
|
|
84
79
|
str,
|
|
85
80
|
StringConstraints(pattern=VERSION_COMPLIANCE_REGEX),
|
|
86
|
-
_custom_error(lambda _, value: RegexViolationError(value, VERSION_COMPLIANCE_REGEX)
|
|
81
|
+
_custom_error(lambda _, value: RegexViolationError(value, VERSION_COMPLIANCE_REGEX)),
|
|
87
82
|
]
|
|
88
83
|
|
|
89
84
|
|
|
90
85
|
def _property_validation(value: str) -> str:
|
|
91
86
|
if not PATTERNS.property_id_compliance.match(value):
|
|
92
|
-
|
|
87
|
+
raise RegexViolationError(value, PROPERTY_ID_COMPLIANCE_REGEX)
|
|
93
88
|
if PATTERNS.more_than_one_alphanumeric.search(value):
|
|
94
89
|
warnings.warn(
|
|
95
90
|
RegexViolationWarning(value, PROPERTY_ID_COMPLIANCE_REGEX, "property", "MoreThanOneNonAlphanumeric"),
|
|
@@ -6,8 +6,6 @@ from pydantic.main import IncEx
|
|
|
6
6
|
from rdflib import Namespace
|
|
7
7
|
|
|
8
8
|
from cognite.neat.constants import get_default_prefixes
|
|
9
|
-
from cognite.neat.issues import MultiValueError
|
|
10
|
-
from cognite.neat.rules import issues
|
|
11
9
|
from cognite.neat.rules.models._base import BaseRules, RoleTypes, SheetList
|
|
12
10
|
from cognite.neat.rules.models.domain import DomainRules
|
|
13
11
|
from cognite.neat.rules.models.entities import (
|
|
@@ -109,7 +107,7 @@ class AssetRules(BaseRules):
|
|
|
109
107
|
if issue_list.warnings:
|
|
110
108
|
issue_list.trigger_warnings()
|
|
111
109
|
if issue_list.has_errors:
|
|
112
|
-
raise
|
|
110
|
+
raise issue_list.as_exception()
|
|
113
111
|
return self
|
|
114
112
|
|
|
115
113
|
def dump(
|
|
@@ -2,7 +2,7 @@ from graphlib import CycleError
|
|
|
2
2
|
from typing import cast
|
|
3
3
|
|
|
4
4
|
from cognite.neat.issues import IssueList
|
|
5
|
-
from cognite.neat.
|
|
5
|
+
from cognite.neat.issues.errors import NeatValueError, PropertyDefinitionError
|
|
6
6
|
from cognite.neat.rules.models._base import SheetList
|
|
7
7
|
from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
|
|
8
8
|
from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity
|
|
@@ -17,7 +17,6 @@ class AssetPostValidation(InformationPostValidation):
|
|
|
17
17
|
return self.issue_list
|
|
18
18
|
|
|
19
19
|
def _parent_property_point_to_class(self) -> None:
|
|
20
|
-
class_property_with_data_value_type = []
|
|
21
20
|
for property_ in cast(SheetList[AssetProperty], self.properties):
|
|
22
21
|
for implementation in property_.implementation:
|
|
23
22
|
if (
|
|
@@ -25,12 +24,15 @@ class AssetPostValidation(InformationPostValidation):
|
|
|
25
24
|
and implementation.property_ == AssetFields.parentExternalId
|
|
26
25
|
and not isinstance(property_.value_type, ClassEntity)
|
|
27
26
|
):
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
self.issue_list.append(
|
|
28
|
+
PropertyDefinitionError(
|
|
29
|
+
property_.class_,
|
|
30
|
+
"class",
|
|
31
|
+
property_.property_,
|
|
32
|
+
"parentExternalId is only allowed to "
|
|
33
|
+
f"point to a Class not {type(property_.value_type).__name__}",
|
|
34
|
+
)
|
|
35
|
+
)
|
|
34
36
|
|
|
35
37
|
def _circular_dependency(self) -> None:
|
|
36
38
|
from cognite.neat.rules.analysis import AssetAnalysis
|
|
@@ -38,4 +40,6 @@ class AssetPostValidation(InformationPostValidation):
|
|
|
38
40
|
try:
|
|
39
41
|
_ = AssetAnalysis(cast(AssetRules, self.rules)).class_topological_sort()
|
|
40
42
|
except CycleError as error:
|
|
41
|
-
self.issue_list.append(
|
|
43
|
+
self.issue_list.append(
|
|
44
|
+
NeatValueError(f"Invalid Asset Hierarchy, circular dependency detected: {error.args[1]}")
|
|
45
|
+
)
|
|
@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, cast
|
|
|
3
3
|
|
|
4
4
|
from rdflib import Namespace
|
|
5
5
|
|
|
6
|
-
from cognite.neat.
|
|
6
|
+
from cognite.neat.issues.warnings.user_modeling import ParentInDifferentSpaceWarning
|
|
7
7
|
from cognite.neat.rules.models._base import SheetList
|
|
8
8
|
from cognite.neat.rules.models.data_types import DataType
|
|
9
9
|
from cognite.neat.rules.models.domain import DomainRules
|
|
@@ -123,9 +123,7 @@ class _DMSRulesConverter:
|
|
|
123
123
|
return None
|
|
124
124
|
if len(parents_other_namespace) > 1:
|
|
125
125
|
warnings.warn(
|
|
126
|
-
|
|
127
|
-
view_id=view.view.as_id(), implements=[v.as_id() for v in parents_other_namespace]
|
|
128
|
-
),
|
|
126
|
+
ParentInDifferentSpaceWarning(view.view.as_id()),
|
|
129
127
|
stacklevel=2,
|
|
130
128
|
)
|
|
131
129
|
other_parent = parents_other_namespace[0]
|
|
@@ -12,7 +12,13 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
12
12
|
ViewPropertyApply,
|
|
13
13
|
)
|
|
14
14
|
|
|
15
|
-
from cognite.neat.
|
|
15
|
+
from cognite.neat.issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
|
|
16
|
+
from cognite.neat.issues.warnings.user_modeling import (
|
|
17
|
+
EmptyContainerWarning,
|
|
18
|
+
HasDataFilterOnNoPropertiesViewWarning,
|
|
19
|
+
HasDataFilterOnViewWithReferencesWarning,
|
|
20
|
+
NodeTypeFilterOnParentViewWarning,
|
|
21
|
+
)
|
|
16
22
|
from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
17
23
|
from cognite.neat.rules.models.data_types import DataType
|
|
18
24
|
from cognite.neat.rules.models.entities import (
|
|
@@ -213,7 +219,11 @@ class _DMSExporter:
|
|
|
213
219
|
if isinstance(view_filter, NodeTypeFilter):
|
|
214
220
|
unique_node_types.update(view_filter.nodes)
|
|
215
221
|
if view.as_id() in parent_views:
|
|
216
|
-
warnings.warn(
|
|
222
|
+
warnings.warn(
|
|
223
|
+
NodeTypeFilterOnParentViewWarning(view.as_id()),
|
|
224
|
+
stacklevel=2,
|
|
225
|
+
)
|
|
226
|
+
|
|
217
227
|
elif isinstance(view_filter, HasDataFilter) and data_model_type == DataModelType.solution:
|
|
218
228
|
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
219
229
|
references = {dms_view.reference.as_view_id()}
|
|
@@ -226,7 +236,8 @@ class _DMSExporter:
|
|
|
226
236
|
else:
|
|
227
237
|
continue
|
|
228
238
|
warnings.warn(
|
|
229
|
-
|
|
239
|
+
HasDataFilterOnViewWithReferencesWarning(view.as_id(), frozenset(references)),
|
|
240
|
+
stacklevel=2,
|
|
230
241
|
)
|
|
231
242
|
|
|
232
243
|
if data_model_type == DataModelType.enterprise:
|
|
@@ -273,7 +284,10 @@ class _DMSExporter:
|
|
|
273
284
|
for container in containers:
|
|
274
285
|
container_id = container.as_id()
|
|
275
286
|
if not (container_properties := container_properties_by_id.get(container_id)):
|
|
276
|
-
warnings.warn(
|
|
287
|
+
warnings.warn(
|
|
288
|
+
EmptyContainerWarning(container_id),
|
|
289
|
+
stacklevel=2,
|
|
290
|
+
)
|
|
277
291
|
container_to_drop.add(container_id)
|
|
278
292
|
continue
|
|
279
293
|
for prop in container_properties:
|
|
@@ -409,7 +423,10 @@ class _DMSExporter:
|
|
|
409
423
|
if not ref_containers or selected_filter_name == HasDataFilter.name:
|
|
410
424
|
# Child filter without container properties
|
|
411
425
|
if selected_filter_name == HasDataFilter.name:
|
|
412
|
-
warnings.warn(
|
|
426
|
+
warnings.warn(
|
|
427
|
+
HasDataFilterOnNoPropertiesViewWarning(view.as_id()),
|
|
428
|
+
stacklevel=2,
|
|
429
|
+
)
|
|
413
430
|
return NodeTypeFilter(inner=[DMSNodeEntity(space=view.space, externalId=view.external_id)])
|
|
414
431
|
else:
|
|
415
432
|
# HasData or not provided (this is the default)
|
|
@@ -493,7 +510,13 @@ class _DMSExporter:
|
|
|
493
510
|
|
|
494
511
|
if reverse_prop is None:
|
|
495
512
|
warnings.warn(
|
|
496
|
-
|
|
513
|
+
PropertyNotFoundWarning(
|
|
514
|
+
source_view_id,
|
|
515
|
+
"view",
|
|
516
|
+
reverse_prop_id or "MISSING",
|
|
517
|
+
dm.PropertyId(prop.view.as_id(), prop.view_property),
|
|
518
|
+
"view property",
|
|
519
|
+
),
|
|
497
520
|
stacklevel=2,
|
|
498
521
|
)
|
|
499
522
|
|
|
@@ -523,7 +546,6 @@ class _DMSExporter:
|
|
|
523
546
|
|
|
524
547
|
elif prop.view and prop.view_property and prop.connection:
|
|
525
548
|
warnings.warn(
|
|
526
|
-
|
|
527
|
-
stacklevel=2,
|
|
549
|
+
NotSupportedWarning(f"{prop.connection} in {prop.view.as_id()!r}.{prop.view_property}"), stacklevel=2
|
|
528
550
|
)
|
|
529
551
|
return None
|
|
@@ -11,8 +11,11 @@ from pydantic import Field, field_serializer, field_validator, model_validator
|
|
|
11
11
|
from pydantic.main import IncEx
|
|
12
12
|
from pydantic_core.core_schema import ValidationInfo
|
|
13
13
|
|
|
14
|
-
from cognite.neat.
|
|
15
|
-
from cognite.neat.
|
|
14
|
+
from cognite.neat.issues import MultiValueError
|
|
15
|
+
from cognite.neat.issues.warnings import (
|
|
16
|
+
PrincipleMatchingSpaceAndVersionWarning,
|
|
17
|
+
PrincipleSolutionBuildsOnEnterpriseWarning,
|
|
18
|
+
)
|
|
16
19
|
from cognite.neat.rules.models._base import (
|
|
17
20
|
BaseMetadata,
|
|
18
21
|
BaseRules,
|
|
@@ -320,8 +323,9 @@ class DMSRules(BaseRules):
|
|
|
320
323
|
raise ValueError("Reference rules cannot have a reference")
|
|
321
324
|
if value.metadata.data_model_type == DataModelType.solution and (metadata := info.data.get("metadata")):
|
|
322
325
|
warnings.warn(
|
|
323
|
-
|
|
324
|
-
metadata.as_data_model_id()
|
|
326
|
+
PrincipleSolutionBuildsOnEnterpriseWarning(
|
|
327
|
+
f"The solution model {metadata.as_data_model_id()} is referencing another "
|
|
328
|
+
f"solution model {value.metadata.as_data_model_id()}",
|
|
325
329
|
),
|
|
326
330
|
stacklevel=2,
|
|
327
331
|
)
|
|
@@ -333,9 +337,21 @@ class DMSRules(BaseRules):
|
|
|
333
337
|
return value
|
|
334
338
|
model_version = metadata.version
|
|
335
339
|
if different_version := [view.view.as_id() for view in value if view.view.version != model_version]:
|
|
336
|
-
|
|
340
|
+
for view_id in different_version:
|
|
341
|
+
warnings.warn(
|
|
342
|
+
PrincipleMatchingSpaceAndVersionWarning(
|
|
343
|
+
f"The view {view_id!r} has a different version than the data model version, {model_version}",
|
|
344
|
+
),
|
|
345
|
+
stacklevel=2,
|
|
346
|
+
)
|
|
337
347
|
if different_space := [view.view.as_id() for view in value if view.view.space != metadata.space]:
|
|
338
|
-
|
|
348
|
+
for view_id in different_space:
|
|
349
|
+
warnings.warn(
|
|
350
|
+
PrincipleMatchingSpaceAndVersionWarning(
|
|
351
|
+
f"The view {view_id!r} is in a different space than the data model space, {metadata.space}",
|
|
352
|
+
),
|
|
353
|
+
stacklevel=2,
|
|
354
|
+
)
|
|
339
355
|
return value
|
|
340
356
|
|
|
341
357
|
@model_validator(mode="after")
|
|
@@ -346,7 +362,7 @@ class DMSRules(BaseRules):
|
|
|
346
362
|
if issue_list.warnings:
|
|
347
363
|
issue_list.trigger_warnings()
|
|
348
364
|
if issue_list.has_errors:
|
|
349
|
-
raise MultiValueError(
|
|
365
|
+
raise MultiValueError(issue_list.errors)
|
|
350
366
|
return self
|
|
351
367
|
|
|
352
368
|
def dump(
|