cognite-neat 0.88.1__py3-none-any.whl → 0.88.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_version.py +1 -1
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/loaders/_base.py +6 -6
- cognite/neat/graph/loaders/_rdf2asset.py +28 -31
- cognite/neat/graph/loaders/_rdf2dms.py +24 -15
- cognite/neat/issues/__init__.py +14 -0
- cognite/neat/issues/_base.py +415 -0
- cognite/neat/issues/errors/__init__.py +72 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +28 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/{rules/issues → issues}/formatters.py +10 -10
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +10 -2
- cognite/neat/rules/exporters/_base.py +6 -6
- cognite/neat/rules/exporters/_rules2dms.py +19 -11
- cognite/neat/rules/exporters/_rules2excel.py +4 -4
- cognite/neat/rules/exporters/_rules2ontology.py +74 -51
- cognite/neat/rules/exporters/_rules2yaml.py +3 -3
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/__init__.py +7 -3
- cognite/neat/rules/importers/_base.py +9 -13
- cognite/neat/rules/importers/_dms2rules.py +42 -24
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +49 -53
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +31 -23
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
- cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +24 -18
- cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +9 -9
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +58 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +60 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
- cognite/neat/rules/importers/_rdf/_shared.py +586 -0
- cognite/neat/rules/importers/_spreadsheet2rules.py +35 -22
- cognite/neat/rules/importers/_yaml2rules.py +23 -21
- cognite/neat/rules/models/_constants.py +2 -1
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +9 -11
- cognite/neat/rules/models/asset/_rules.py +1 -3
- cognite/neat/rules/models/asset/_validation.py +14 -10
- cognite/neat/rules/models/dms/_converter.py +2 -4
- cognite/neat/rules/models/dms/_exporter.py +30 -8
- cognite/neat/rules/models/dms/_rules.py +23 -7
- cognite/neat/rules/models/dms/_schema.py +94 -62
- cognite/neat/rules/models/dms/_validation.py +105 -66
- cognite/neat/rules/models/entities.py +3 -0
- cognite/neat/rules/models/information/_converter.py +2 -2
- cognite/neat/rules/models/information/_rules.py +7 -8
- cognite/neat/rules/models/information/_validation.py +48 -25
- cognite/neat/rules/transformers/__init__.py +0 -0
- cognite/neat/rules/transformers/_base.py +15 -0
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -10
- cognite/neat/workflows/steps/lib/current/rules_importer.py +78 -6
- cognite/neat/workflows/steps/lib/current/rules_validator.py +20 -9
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/RECORD +86 -77
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/graph/issues/loader.py +0 -104
- cognite/neat/issues.py +0 -158
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -209
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
- cognite/neat/rules/issues/__init__.py +0 -26
- cognite/neat/rules/issues/base.py +0 -82
- cognite/neat/rules/issues/dms.py +0 -683
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/importing.py +0 -423
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{graph/issues → rules/importers/_rdf}/__init__.py +0 -0
- /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- /cognite/neat/{graph/stores → store}/_provenance.py +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/entry_points.txt +0 -0
|
@@ -5,7 +5,7 @@ from typing import Generic, TypeVar
|
|
|
5
5
|
|
|
6
6
|
from cognite.client import CogniteClient
|
|
7
7
|
|
|
8
|
-
from cognite.neat.rules._shared import
|
|
8
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
9
9
|
from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
|
|
10
10
|
from cognite.neat.utils.auxiliary import class_html_doc
|
|
11
11
|
from cognite.neat.utils.upload import UploadResult, UploadResultList
|
|
@@ -18,14 +18,14 @@ class BaseExporter(ABC, Generic[T_Export]):
|
|
|
18
18
|
_encoding = "utf-8"
|
|
19
19
|
|
|
20
20
|
@abstractmethod
|
|
21
|
-
def export_to_file(self, rules:
|
|
21
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
22
22
|
raise NotImplementedError
|
|
23
23
|
|
|
24
24
|
@abstractmethod
|
|
25
|
-
def export(self, rules:
|
|
25
|
+
def export(self, rules: VerifiedRules) -> T_Export:
|
|
26
26
|
raise NotImplementedError
|
|
27
27
|
|
|
28
|
-
def _convert_to_output_role(self, rules:
|
|
28
|
+
def _convert_to_output_role(self, rules: VerifiedRules, output_role: RoleTypes | None = None) -> VerifiedRules:
|
|
29
29
|
if rules.metadata.role is output_role or output_role is None:
|
|
30
30
|
return rules
|
|
31
31
|
elif output_role is RoleTypes.dms and isinstance(rules, InformationRules):
|
|
@@ -43,9 +43,9 @@ class BaseExporter(ABC, Generic[T_Export]):
|
|
|
43
43
|
class CDFExporter(BaseExporter[T_Export]):
|
|
44
44
|
@abstractmethod
|
|
45
45
|
def export_to_cdf_iterable(
|
|
46
|
-
self, rules:
|
|
46
|
+
self, rules: VerifiedRules, client: CogniteClient, dry_run: bool = False
|
|
47
47
|
) -> Iterable[UploadResult]:
|
|
48
48
|
raise NotImplementedError
|
|
49
49
|
|
|
50
|
-
def export_to_cdf(self, rules:
|
|
50
|
+
def export_to_cdf(self, rules: VerifiedRules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
|
|
51
51
|
return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))
|
|
@@ -15,9 +15,12 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
15
15
|
)
|
|
16
16
|
from cognite.client.exceptions import CogniteAPIError
|
|
17
17
|
|
|
18
|
-
from cognite.neat.
|
|
19
|
-
from cognite.neat.
|
|
20
|
-
|
|
18
|
+
from cognite.neat.issues import IssueList
|
|
19
|
+
from cognite.neat.issues.warnings import (
|
|
20
|
+
PrincipleOneModelOneSpaceWarning,
|
|
21
|
+
ResourceRetrievalWarning,
|
|
22
|
+
)
|
|
23
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
21
24
|
from cognite.neat.rules.models import InformationRules
|
|
22
25
|
from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
|
|
23
26
|
from cognite.neat.utils.cdf.loaders import (
|
|
@@ -79,7 +82,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
79
82
|
self.suppress_warnings = suppress_warnings
|
|
80
83
|
self._schema: DMSSchema | None = None
|
|
81
84
|
|
|
82
|
-
def export_to_file(self, rules:
|
|
85
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
83
86
|
"""Export the rules to a file(s).
|
|
84
87
|
|
|
85
88
|
If the file is a directory, the components will be exported to separate files, otherwise they will be
|
|
@@ -94,12 +97,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
94
97
|
else:
|
|
95
98
|
self._export_to_zip_file(filepath, rules)
|
|
96
99
|
|
|
97
|
-
def _export_to_directory(self, directory: Path, rules:
|
|
100
|
+
def _export_to_directory(self, directory: Path, rules: VerifiedRules) -> None:
|
|
98
101
|
schema = self.export(rules)
|
|
99
102
|
exclude = self._create_exclude_set()
|
|
100
103
|
schema.to_directory(directory, exclude=exclude, new_line=self._new_line, encoding=self._encoding)
|
|
101
104
|
|
|
102
|
-
def _export_to_zip_file(self, filepath: Path, rules:
|
|
105
|
+
def _export_to_zip_file(self, filepath: Path, rules: VerifiedRules) -> None:
|
|
103
106
|
if filepath.suffix not in {".zip"}:
|
|
104
107
|
warnings.warn("File extension is not .zip, adding it to the file name", stacklevel=2)
|
|
105
108
|
filepath = filepath.with_suffix(".zip")
|
|
@@ -114,7 +117,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
114
117
|
exclude = {"spaces", "data_models", "views", "containers", "node_types"} - self.export_components
|
|
115
118
|
return exclude
|
|
116
119
|
|
|
117
|
-
def export(self, rules:
|
|
120
|
+
def export(self, rules: VerifiedRules) -> DMSSchema:
|
|
118
121
|
if isinstance(rules, DMSRules):
|
|
119
122
|
dms_rules = rules
|
|
120
123
|
elif isinstance(rules, InformationRules):
|
|
@@ -123,7 +126,9 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
123
126
|
raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
|
|
124
127
|
return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
|
|
125
128
|
|
|
126
|
-
def delete_from_cdf(
|
|
129
|
+
def delete_from_cdf(
|
|
130
|
+
self, rules: VerifiedRules, client: CogniteClient, dry_run: bool = False
|
|
131
|
+
) -> Iterable[UploadResult]:
|
|
127
132
|
to_export = self._prepare_exporters(rules, client)
|
|
128
133
|
|
|
129
134
|
# we need to reverse order in which we are picking up the items to delete
|
|
@@ -167,7 +172,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
167
172
|
)
|
|
168
173
|
|
|
169
174
|
def export_to_cdf_iterable(
|
|
170
|
-
self, rules:
|
|
175
|
+
self, rules: VerifiedRules, client: CogniteClient, dry_run: bool = False
|
|
171
176
|
) -> Iterable[UploadResult]:
|
|
172
177
|
to_export = self._prepare_exporters(rules, client)
|
|
173
178
|
|
|
@@ -297,7 +302,10 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
297
302
|
if isinstance(loader, DataModelLoader):
|
|
298
303
|
models = cast(list[DataModelApply], items)
|
|
299
304
|
if other_models := self._exist_other_data_models(loader, models):
|
|
300
|
-
warning =
|
|
305
|
+
warning = PrincipleOneModelOneSpaceWarning(
|
|
306
|
+
f"There are multiple data models in the same space {models[0].space}. "
|
|
307
|
+
f"Other data models in the space are {other_models}.",
|
|
308
|
+
)
|
|
301
309
|
if not self.suppress_warnings:
|
|
302
310
|
warnings.warn(warning, stacklevel=2)
|
|
303
311
|
issue_list.append(warning)
|
|
@@ -313,7 +321,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
313
321
|
try:
|
|
314
322
|
data_models = loader.client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
|
|
315
323
|
except CogniteAPIError as e:
|
|
316
|
-
warnings.warn(
|
|
324
|
+
warnings.warn(ResourceRetrievalWarning(frozenset({space}), "space", str(e)), stacklevel=2)
|
|
317
325
|
return []
|
|
318
326
|
else:
|
|
319
327
|
return [
|
|
@@ -12,7 +12,7 @@ from openpyxl.cell import MergedCell
|
|
|
12
12
|
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
13
13
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
14
14
|
|
|
15
|
-
from cognite.neat.rules._shared import
|
|
15
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
16
16
|
from cognite.neat.rules.models import (
|
|
17
17
|
DataModelType,
|
|
18
18
|
ExtensionCategory,
|
|
@@ -89,7 +89,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
89
89
|
self.new_model_id = new_model_id
|
|
90
90
|
self.dump_as = dump_as
|
|
91
91
|
|
|
92
|
-
def export_to_file(self, rules:
|
|
92
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
93
93
|
"""Exports transformation rules to excel file."""
|
|
94
94
|
data = self.export(rules)
|
|
95
95
|
try:
|
|
@@ -98,7 +98,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
98
98
|
data.close()
|
|
99
99
|
return None
|
|
100
100
|
|
|
101
|
-
def export(self, rules:
|
|
101
|
+
def export(self, rules: VerifiedRules) -> Workbook:
|
|
102
102
|
rules = self._convert_to_output_role(rules, self.output_role)
|
|
103
103
|
workbook = Workbook()
|
|
104
104
|
# Remove default sheet named "Sheet"
|
|
@@ -147,7 +147,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
147
147
|
self,
|
|
148
148
|
workbook: Workbook,
|
|
149
149
|
dumped_rules: dict[str, Any],
|
|
150
|
-
rules:
|
|
150
|
+
rules: VerifiedRules,
|
|
151
151
|
sheet_prefix: str = "",
|
|
152
152
|
):
|
|
153
153
|
for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
|
|
@@ -9,19 +9,12 @@ from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, BNode, Graph, Literal, Namespac
|
|
|
9
9
|
from rdflib.collection import Collection as GraphCollection
|
|
10
10
|
|
|
11
11
|
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
12
|
-
from cognite.neat.
|
|
13
|
-
from cognite.neat.
|
|
14
|
-
|
|
15
|
-
MissingDataModelPrefixOrNamespaceWarning,
|
|
16
|
-
OntologyMultiDefinitionPropertyWarning,
|
|
17
|
-
OntologyMultiDomainPropertyWarning,
|
|
18
|
-
OntologyMultiLabeledPropertyWarning,
|
|
19
|
-
OntologyMultiRangePropertyWarning,
|
|
20
|
-
OntologyMultiTypePropertyWarning,
|
|
21
|
-
PrefixMissingError,
|
|
22
|
-
PropertiesDefinedMultipleTimesError,
|
|
23
|
-
PropertyDefinitionsNotForSamePropertyError,
|
|
12
|
+
from cognite.neat.issues import MultiValueError
|
|
13
|
+
from cognite.neat.issues.errors import (
|
|
14
|
+
PropertyDefinitionDuplicatedError,
|
|
24
15
|
)
|
|
16
|
+
from cognite.neat.issues.warnings import PropertyDefinitionDuplicatedWarning
|
|
17
|
+
from cognite.neat.rules.analysis import InformationAnalysis
|
|
25
18
|
from cognite.neat.rules.models import DMSRules
|
|
26
19
|
from cognite.neat.rules.models.data_types import DataType
|
|
27
20
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
@@ -31,43 +24,42 @@ from cognite.neat.rules.models.information import (
|
|
|
31
24
|
InformationProperty,
|
|
32
25
|
InformationRules,
|
|
33
26
|
)
|
|
34
|
-
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
35
27
|
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
36
28
|
|
|
37
29
|
from ._base import BaseExporter
|
|
38
|
-
from ._validation import
|
|
30
|
+
from ._validation import duplicated_properties
|
|
39
31
|
|
|
40
32
|
if sys.version_info >= (3, 11):
|
|
41
33
|
from typing import Self
|
|
42
34
|
else:
|
|
43
35
|
from typing_extensions import Self
|
|
44
36
|
|
|
45
|
-
from cognite.neat.rules._shared import
|
|
37
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
46
38
|
|
|
47
39
|
|
|
48
40
|
class GraphExporter(BaseExporter[Graph], ABC):
|
|
49
|
-
def export_to_file(self, rules:
|
|
41
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
50
42
|
self.export(rules).serialize(destination=filepath, encoding=self._encoding, newline=self._new_line)
|
|
51
43
|
|
|
52
44
|
|
|
53
45
|
class OWLExporter(GraphExporter):
|
|
54
46
|
"""Exports rules to an OWL ontology."""
|
|
55
47
|
|
|
56
|
-
def export(self, rules:
|
|
48
|
+
def export(self, rules: VerifiedRules) -> Graph:
|
|
57
49
|
return Ontology.from_rules(rules).as_owl()
|
|
58
50
|
|
|
59
51
|
|
|
60
52
|
class SHACLExporter(GraphExporter):
|
|
61
53
|
"""Exports rules to a SHACL graph."""
|
|
62
54
|
|
|
63
|
-
def export(self, rules:
|
|
55
|
+
def export(self, rules: VerifiedRules) -> Graph:
|
|
64
56
|
return Ontology.from_rules(rules).as_shacl()
|
|
65
57
|
|
|
66
58
|
|
|
67
59
|
class SemanticDataModelExporter(GraphExporter):
|
|
68
60
|
"""Exports rules to a semantic data model."""
|
|
69
61
|
|
|
70
|
-
def export(self, rules:
|
|
62
|
+
def export(self, rules: VerifiedRules) -> Graph:
|
|
71
63
|
return Ontology.from_rules(rules).as_semantic_data_model()
|
|
72
64
|
|
|
73
65
|
|
|
@@ -94,7 +86,7 @@ class Ontology(OntologyModel):
|
|
|
94
86
|
prefixes: dict[str, Namespace]
|
|
95
87
|
|
|
96
88
|
@classmethod
|
|
97
|
-
def from_rules(cls, input_rules:
|
|
89
|
+
def from_rules(cls, input_rules: VerifiedRules) -> Self:
|
|
98
90
|
"""
|
|
99
91
|
Generates an ontology from a set of transformation rules.
|
|
100
92
|
|
|
@@ -111,17 +103,20 @@ class Ontology(OntologyModel):
|
|
|
111
103
|
else:
|
|
112
104
|
raise ValueError(f"{type(input_rules).__name__} cannot be exported to Ontology")
|
|
113
105
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
106
|
+
if duplicates := duplicated_properties(rules.properties):
|
|
107
|
+
errors = []
|
|
108
|
+
for (class_, property_), definitions in duplicates.items():
|
|
109
|
+
errors.append(
|
|
110
|
+
PropertyDefinitionDuplicatedError(
|
|
111
|
+
class_,
|
|
112
|
+
"class",
|
|
113
|
+
property_,
|
|
114
|
+
frozenset({str(definition[1].value_type) for definition in definitions}),
|
|
115
|
+
tuple(definition[0] for definition in definitions),
|
|
116
|
+
"rows",
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
raise MultiValueError(errors)
|
|
125
120
|
|
|
126
121
|
class_dict = InformationAnalysis(rules).as_class_dict()
|
|
127
122
|
return cls(
|
|
@@ -184,9 +179,6 @@ class Ontology(OntologyModel):
|
|
|
184
179
|
for prefix, namespace in self.prefixes.items():
|
|
185
180
|
owl.bind(prefix, namespace)
|
|
186
181
|
|
|
187
|
-
if self.metadata.namespace is None:
|
|
188
|
-
raise MetadataSheetNamespaceNotDefinedError().as_exception()
|
|
189
|
-
|
|
190
182
|
owl.add((URIRef(self.metadata.namespace), RDF.type, OWL.Ontology))
|
|
191
183
|
for property_ in self.properties:
|
|
192
184
|
for triple in property_.triples:
|
|
@@ -233,8 +225,6 @@ class OWLMetadata(InformationMetadata):
|
|
|
233
225
|
@property
|
|
234
226
|
def triples(self) -> list[tuple]:
|
|
235
227
|
# Mandatory triples originating from Metadata mandatory fields
|
|
236
|
-
if self.namespace is None:
|
|
237
|
-
raise MetadataSheetNamespaceNotDefinedError().as_exception()
|
|
238
228
|
triples: list[tuple] = [
|
|
239
229
|
(URIRef(self.namespace), DCTERMS.hasVersion, Literal(self.version)),
|
|
240
230
|
(URIRef(self.namespace), OWL.versionInfo, Literal(self.version)),
|
|
@@ -323,16 +313,17 @@ class OWLProperty(OntologyModel):
|
|
|
323
313
|
range_: set[URIRef]
|
|
324
314
|
namespace: Namespace
|
|
325
315
|
|
|
326
|
-
@staticmethod
|
|
327
|
-
def same_property_id(definitions: list[InformationProperty]) -> bool:
|
|
328
|
-
return len({definition.property_ for definition in definitions}) == 1
|
|
329
|
-
|
|
330
316
|
@classmethod
|
|
331
317
|
def from_list_of_properties(cls, definitions: list[InformationProperty], namespace: Namespace) -> "OWLProperty":
|
|
332
318
|
"""Here list of properties is a list of properties with the same id, but different definitions."""
|
|
333
|
-
|
|
334
|
-
if
|
|
335
|
-
raise
|
|
319
|
+
property_ids = {definition.property_ for definition in definitions}
|
|
320
|
+
if len(property_ids) != 1:
|
|
321
|
+
raise PropertyDefinitionDuplicatedError(
|
|
322
|
+
definitions[0].class_,
|
|
323
|
+
"class",
|
|
324
|
+
definitions[0].property_,
|
|
325
|
+
frozenset(property_ids),
|
|
326
|
+
)
|
|
336
327
|
|
|
337
328
|
owl_property = cls.model_construct(
|
|
338
329
|
id_=namespace[definitions[0].property_],
|
|
@@ -363,8 +354,15 @@ class OWLProperty(OntologyModel):
|
|
|
363
354
|
def is_multi_type(cls, v, info: ValidationInfo):
|
|
364
355
|
if len(v) > 1:
|
|
365
356
|
warnings.warn(
|
|
366
|
-
|
|
367
|
-
remove_namespace_from_uri(info.data["
|
|
357
|
+
PropertyDefinitionDuplicatedWarning(
|
|
358
|
+
remove_namespace_from_uri(info.data["id"]),
|
|
359
|
+
"class",
|
|
360
|
+
"type",
|
|
361
|
+
frozenset({remove_namespace_from_uri(t) for t in v}),
|
|
362
|
+
"This warning occurs when a same property is define for two object/classes where"
|
|
363
|
+
" its expected value type is different in one definition, e.g. acts as an edge, while in "
|
|
364
|
+
"other definition acts as and attribute",
|
|
365
|
+
"If a property takes different value types for different objects, simply define new property",
|
|
368
366
|
),
|
|
369
367
|
stacklevel=2,
|
|
370
368
|
)
|
|
@@ -374,8 +372,14 @@ class OWLProperty(OntologyModel):
|
|
|
374
372
|
def is_multi_range(cls, v, info: ValidationInfo):
|
|
375
373
|
if len(v) > 1:
|
|
376
374
|
warnings.warn(
|
|
377
|
-
|
|
378
|
-
remove_namespace_from_uri(info.data["id_"]),
|
|
375
|
+
PropertyDefinitionDuplicatedWarning(
|
|
376
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
377
|
+
"class",
|
|
378
|
+
"range",
|
|
379
|
+
frozenset({remove_namespace_from_uri(t) for t in v}),
|
|
380
|
+
"This warning occurs when a property takes range of "
|
|
381
|
+
"values which consists of union of multiple value types.",
|
|
382
|
+
"If value types for different objects, simply define new property",
|
|
379
383
|
),
|
|
380
384
|
stacklevel=2,
|
|
381
385
|
)
|
|
@@ -385,8 +389,15 @@ class OWLProperty(OntologyModel):
|
|
|
385
389
|
def is_multi_domain(cls, v, info: ValidationInfo):
|
|
386
390
|
if len(v) > 1:
|
|
387
391
|
warnings.warn(
|
|
388
|
-
|
|
389
|
-
remove_namespace_from_uri(info.data["id_"]),
|
|
392
|
+
PropertyDefinitionDuplicatedWarning(
|
|
393
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
394
|
+
"class",
|
|
395
|
+
"domain",
|
|
396
|
+
frozenset({remove_namespace_from_uri(t) for t in v}),
|
|
397
|
+
"This warning occurs when a same property is define for two object/classes where"
|
|
398
|
+
" its expected value type is different in one definition, e.g. acts as an edge, while in "
|
|
399
|
+
"other definition acts as and attribute",
|
|
400
|
+
"If value types for different objects, simply define new property",
|
|
390
401
|
),
|
|
391
402
|
stacklevel=2,
|
|
392
403
|
)
|
|
@@ -396,7 +407,13 @@ class OWLProperty(OntologyModel):
|
|
|
396
407
|
def has_multi_name(cls, v, info: ValidationInfo):
|
|
397
408
|
if len(v) > 1:
|
|
398
409
|
warnings.warn(
|
|
399
|
-
|
|
410
|
+
PropertyDefinitionDuplicatedWarning(
|
|
411
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
412
|
+
"class",
|
|
413
|
+
"label",
|
|
414
|
+
frozenset(v),
|
|
415
|
+
f"Only the first label (name) will be used, {v[0]}",
|
|
416
|
+
),
|
|
400
417
|
stacklevel=2,
|
|
401
418
|
)
|
|
402
419
|
return v
|
|
@@ -405,7 +422,13 @@ class OWLProperty(OntologyModel):
|
|
|
405
422
|
def has_multi_comment(cls, v, info: ValidationInfo):
|
|
406
423
|
if len(v) > 1:
|
|
407
424
|
warnings.warn(
|
|
408
|
-
|
|
425
|
+
PropertyDefinitionDuplicatedWarning(
|
|
426
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
427
|
+
"class",
|
|
428
|
+
"comment",
|
|
429
|
+
frozenset(v),
|
|
430
|
+
"All definitions will be concatenated to form a single definition.",
|
|
431
|
+
),
|
|
409
432
|
stacklevel=2,
|
|
410
433
|
)
|
|
411
434
|
return v
|
|
@@ -5,7 +5,7 @@ from typing import Literal, get_args
|
|
|
5
5
|
|
|
6
6
|
import yaml
|
|
7
7
|
|
|
8
|
-
from cognite.neat.rules._shared import
|
|
8
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
9
9
|
from cognite.neat.rules.models import RoleTypes
|
|
10
10
|
|
|
11
11
|
from ._base import BaseExporter
|
|
@@ -47,7 +47,7 @@ class YAMLExporter(BaseExporter[str]):
|
|
|
47
47
|
self.output = output
|
|
48
48
|
self.output_role = output_role
|
|
49
49
|
|
|
50
|
-
def export_to_file(self, rules:
|
|
50
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
51
51
|
"""Exports transformation rules to YAML/JSON file(s)."""
|
|
52
52
|
if self.files == "single":
|
|
53
53
|
if filepath.suffix != f".{self.output}":
|
|
@@ -57,7 +57,7 @@ class YAMLExporter(BaseExporter[str]):
|
|
|
57
57
|
else:
|
|
58
58
|
raise NotImplementedError(f"Exporting to {self.files} files is not supported")
|
|
59
59
|
|
|
60
|
-
def export(self, rules:
|
|
60
|
+
def export(self, rules: VerifiedRules) -> str:
|
|
61
61
|
"""Export rules to YAML (or JSON) format.
|
|
62
62
|
|
|
63
63
|
Args:
|
|
@@ -1,99 +1,14 @@
|
|
|
1
|
-
import
|
|
2
|
-
from
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from collections.abc import Iterable
|
|
3
3
|
|
|
4
|
-
from cognite.neat.
|
|
5
|
-
from cognite.neat.rules.
|
|
6
|
-
from cognite.neat.rules.issues.importing import PropertyRedefinedWarning
|
|
7
|
-
from cognite.neat.rules.models import InformationRules
|
|
8
|
-
from cognite.neat.utils.regex_patterns import DMS_PROPERTY_ID_COMPLIANCE_REGEX, PATTERNS, VIEW_ID_COMPLIANCE_REGEX
|
|
4
|
+
from cognite.neat.rules.models.entities import ClassEntity
|
|
5
|
+
from cognite.neat.rules.models.information import InformationProperty
|
|
9
6
|
|
|
10
7
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def are_entity_names_dms_compliant(rules: InformationRules, return_report: Literal[False] = False) -> bool: ...
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def are_entity_names_dms_compliant(
|
|
22
|
-
rules: InformationRules, return_report: bool = False
|
|
23
|
-
) -> bool | tuple[bool, list[dict]]:
|
|
24
|
-
"""Check if data model definitions are valid."""
|
|
25
|
-
|
|
26
|
-
flag: bool = True
|
|
27
|
-
with warnings.catch_warnings(record=True) as validation_warnings:
|
|
28
|
-
for class_ in rules.classes:
|
|
29
|
-
if not PATTERNS.view_id_compliance.match(class_.class_.suffix):
|
|
30
|
-
warnings.warn(
|
|
31
|
-
EntityIDNotDMSCompliantWarning(class_.class_.versioned_id, "Class", VIEW_ID_COMPLIANCE_REGEX),
|
|
32
|
-
stacklevel=2,
|
|
33
|
-
)
|
|
34
|
-
flag = False
|
|
35
|
-
|
|
36
|
-
for _, property_ in enumerate(rules.properties):
|
|
37
|
-
# check class id which would resolve as view/container id
|
|
38
|
-
if not PATTERNS.view_id_compliance.match(property_.class_.suffix):
|
|
39
|
-
warnings.warn(
|
|
40
|
-
EntityIDNotDMSCompliantWarning(
|
|
41
|
-
property_.class_.versioned_id,
|
|
42
|
-
"Class",
|
|
43
|
-
VIEW_ID_COMPLIANCE_REGEX,
|
|
44
|
-
),
|
|
45
|
-
stacklevel=2,
|
|
46
|
-
)
|
|
47
|
-
flag = False
|
|
48
|
-
|
|
49
|
-
# check property id which would resolve as view/container id
|
|
50
|
-
if not PATTERNS.dms_property_id_compliance.match(property_.property_):
|
|
51
|
-
warnings.warn(
|
|
52
|
-
EntityIDNotDMSCompliantWarning(property_.property_, "Property", DMS_PROPERTY_ID_COMPLIANCE_REGEX),
|
|
53
|
-
stacklevel=2,
|
|
54
|
-
)
|
|
55
|
-
flag = False
|
|
56
|
-
|
|
57
|
-
if return_report:
|
|
58
|
-
return flag, wrangle_warnings(validation_warnings)
|
|
59
|
-
else:
|
|
60
|
-
return flag
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
@overload
|
|
64
|
-
def are_properties_redefined(rules: InformationRules, return_report: Literal[True]) -> tuple[bool, list[dict]]: ...
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
@overload
|
|
68
|
-
def are_properties_redefined(rules: InformationRules, return_report: Literal[False] = False) -> bool: ...
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def are_properties_redefined(rules: InformationRules, return_report: bool = False) -> bool | tuple[bool, list[dict]]:
|
|
72
|
-
flag: bool = False
|
|
73
|
-
with warnings.catch_warnings(record=True) as validation_warnings:
|
|
74
|
-
analyzed_properties: dict[str, list[str]] = {}
|
|
75
|
-
for property_ in rules.properties:
|
|
76
|
-
if property_.property_ not in analyzed_properties:
|
|
77
|
-
analyzed_properties[property_.property_] = [property_.class_.versioned_id]
|
|
78
|
-
elif property_.class_ in analyzed_properties[property_.property_]:
|
|
79
|
-
flag = True
|
|
80
|
-
warnings.warn(
|
|
81
|
-
PropertyRedefinedWarning(property_.property_, property_.class_.versioned_id),
|
|
82
|
-
stacklevel=2,
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
else:
|
|
86
|
-
analyzed_properties[property_.property_].append(property_.class_.versioned_id)
|
|
87
|
-
|
|
88
|
-
if return_report:
|
|
89
|
-
return flag, wrangle_warnings(validation_warnings)
|
|
90
|
-
else:
|
|
91
|
-
return flag
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
def property_ids_camel_case_compliant(rules) -> bool | tuple[bool, list[dict]]:
|
|
95
|
-
raise NotImplementedError()
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def class_id_pascal_case_compliant(rules) -> bool | tuple[bool, list[dict]]:
|
|
99
|
-
raise NotImplementedError()
|
|
8
|
+
def duplicated_properties(
|
|
9
|
+
properties: Iterable[InformationProperty],
|
|
10
|
+
) -> dict[tuple[ClassEntity, str], list[tuple[int, InformationProperty]]]:
|
|
11
|
+
class_properties_by_id: dict[tuple[ClassEntity, str], list[tuple[int, InformationProperty]]] = defaultdict(list)
|
|
12
|
+
for prop_no, prop in enumerate(properties):
|
|
13
|
+
class_properties_by_id[(prop.class_, prop.property_)].append((prop_no, prop))
|
|
14
|
+
return {k: v for k, v in class_properties_by_id.items() if len(v) > 1}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
from ._base import BaseImporter
|
|
2
2
|
from ._dms2rules import DMSImporter
|
|
3
3
|
from ._dtdl2rules import DTDLImporter
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
4
|
+
from ._rdf._imf2rules import IMFImporter
|
|
5
|
+
from ._rdf._inference2rules import InferenceImporter
|
|
6
|
+
from ._rdf._owl2rules import OWLImporter
|
|
6
7
|
from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
|
|
7
8
|
from ._yaml2rules import YAMLImporter
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
"BaseImporter",
|
|
11
12
|
"OWLImporter",
|
|
13
|
+
"IMFImporter",
|
|
12
14
|
"DMSImporter",
|
|
13
15
|
"ExcelImporter",
|
|
14
16
|
"GoogleSheetImporter",
|
|
@@ -25,7 +27,9 @@ def _repr_html_() -> str:
|
|
|
25
27
|
[
|
|
26
28
|
{
|
|
27
29
|
"Importer": name,
|
|
28
|
-
"Description":
|
|
30
|
+
"Description": (
|
|
31
|
+
globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing"
|
|
32
|
+
),
|
|
29
33
|
}
|
|
30
34
|
for name in __all__
|
|
31
35
|
if name != "BaseImporter"
|
|
@@ -9,12 +9,8 @@ from typing import Any, Literal, overload
|
|
|
9
9
|
from pydantic import ValidationError
|
|
10
10
|
from rdflib import Namespace
|
|
11
11
|
|
|
12
|
-
from cognite.neat.
|
|
13
|
-
from cognite.neat.rules.
|
|
14
|
-
IssueList,
|
|
15
|
-
NeatValidationError,
|
|
16
|
-
ValidationWarning,
|
|
17
|
-
)
|
|
12
|
+
from cognite.neat.issues import IssueList, NeatError, NeatWarning
|
|
13
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
18
14
|
from cognite.neat.rules.models import AssetRules, DMSRules, InformationRules, RoleTypes
|
|
19
15
|
from cognite.neat.utils.auxiliary import class_html_doc
|
|
20
16
|
|
|
@@ -25,19 +21,19 @@ class BaseImporter(ABC):
|
|
|
25
21
|
"""
|
|
26
22
|
|
|
27
23
|
@overload
|
|
28
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) ->
|
|
24
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> VerifiedRules: ...
|
|
29
25
|
|
|
30
26
|
@overload
|
|
31
27
|
def to_rules(
|
|
32
28
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
33
|
-
) -> tuple[
|
|
29
|
+
) -> tuple[VerifiedRules | None, IssueList]: ...
|
|
34
30
|
|
|
35
31
|
@abstractmethod
|
|
36
32
|
def to_rules(
|
|
37
33
|
self,
|
|
38
34
|
errors: Literal["raise", "continue"] = "continue",
|
|
39
35
|
role: RoleTypes | None = None,
|
|
40
|
-
) -> tuple[
|
|
36
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
41
37
|
"""
|
|
42
38
|
Creates `Rules` object from the data for target role.
|
|
43
39
|
"""
|
|
@@ -46,11 +42,11 @@ class BaseImporter(ABC):
|
|
|
46
42
|
@classmethod
|
|
47
43
|
def _to_output(
|
|
48
44
|
cls,
|
|
49
|
-
rules:
|
|
45
|
+
rules: VerifiedRules,
|
|
50
46
|
issues: IssueList,
|
|
51
47
|
errors: Literal["raise", "continue"] = "continue",
|
|
52
48
|
role: RoleTypes | None = None,
|
|
53
|
-
) -> tuple[
|
|
49
|
+
) -> tuple[VerifiedRules | None, IssueList] | VerifiedRules:
|
|
54
50
|
"""Converts the rules to the output format."""
|
|
55
51
|
|
|
56
52
|
if rules.metadata.role is role or role is None:
|
|
@@ -103,8 +99,8 @@ class _FutureResult:
|
|
|
103
99
|
@contextmanager
|
|
104
100
|
def _handle_issues(
|
|
105
101
|
issues: IssueList,
|
|
106
|
-
error_cls: type[
|
|
107
|
-
warning_cls: type[
|
|
102
|
+
error_cls: type[NeatError] = NeatError,
|
|
103
|
+
warning_cls: type[NeatWarning] = NeatWarning,
|
|
108
104
|
error_args: dict[str, Any] | None = None,
|
|
109
105
|
) -> Iterator[_FutureResult]:
|
|
110
106
|
"""This is an internal help function to handle issues and warnings.
|