cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +3 -0
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -1
- cognite/neat/graph/loaders/_base.py +3 -3
- cognite/neat/graph/loaders/_rdf2asset.py +24 -25
- cognite/neat/graph/loaders/_rdf2dms.py +20 -15
- cognite/neat/issues/__init__.py +1 -3
- cognite/neat/issues/_base.py +261 -71
- cognite/neat/issues/errors/__init__.py +73 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +35 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/issues/formatters.py +1 -1
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +53 -2
- cognite/neat/rules/analysis/_base.py +1 -1
- cognite/neat/rules/exporters/_base.py +7 -18
- cognite/neat/rules/exporters/_rules2dms.py +17 -20
- cognite/neat/rules/exporters/_rules2excel.py +9 -16
- cognite/neat/rules/exporters/_rules2ontology.py +77 -64
- cognite/neat/rules/exporters/_rules2yaml.py +6 -9
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/_base.py +9 -58
- cognite/neat/rules/importers/_dms2rules.py +188 -135
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +48 -35
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +36 -45
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +8 -4
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +3 -3
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +18 -11
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +12 -19
- cognite/neat/rules/importers/_rdf/_inference2rules.py +14 -37
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +9 -20
- cognite/neat/rules/importers/_rdf/_shared.py +4 -4
- cognite/neat/rules/importers/_spreadsheet2rules.py +46 -97
- cognite/neat/rules/importers/_yaml2rules.py +32 -58
- cognite/neat/rules/models/__init__.py +21 -5
- cognite/neat/rules/models/_base_input.py +162 -0
- cognite/neat/rules/models/{_base.py → _base_rules.py} +1 -12
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/{_types/_field.py → _types.py} +5 -10
- cognite/neat/rules/models/asset/__init__.py +5 -2
- cognite/neat/rules/models/asset/_rules.py +3 -23
- cognite/neat/rules/models/asset/_rules_input.py +40 -115
- cognite/neat/rules/models/asset/_validation.py +14 -10
- cognite/neat/rules/models/data_types.py +150 -44
- cognite/neat/rules/models/dms/__init__.py +19 -7
- cognite/neat/rules/models/dms/_exporter.py +102 -34
- cognite/neat/rules/models/dms/_rules.py +65 -162
- cognite/neat/rules/models/dms/_rules_input.py +186 -254
- cognite/neat/rules/models/dms/_schema.py +87 -78
- cognite/neat/rules/models/dms/_serializer.py +44 -3
- cognite/neat/rules/models/dms/_validation.py +106 -68
- cognite/neat/rules/models/domain.py +52 -1
- cognite/neat/rules/models/entities/__init__.py +63 -0
- cognite/neat/rules/models/entities/_constants.py +73 -0
- cognite/neat/rules/models/entities/_loaders.py +76 -0
- cognite/neat/rules/models/entities/_multi_value.py +67 -0
- cognite/neat/rules/models/{entities.py → entities/_single_value.py} +74 -232
- cognite/neat/rules/models/entities/_types.py +86 -0
- cognite/neat/rules/models/{wrapped_entities.py → entities/_wrapped.py} +1 -1
- cognite/neat/rules/models/information/__init__.py +10 -2
- cognite/neat/rules/models/information/_rules.py +10 -22
- cognite/neat/rules/models/information/_rules_input.py +57 -204
- cognite/neat/rules/models/information/_validation.py +48 -25
- cognite/neat/rules/transformers/__init__.py +21 -0
- cognite/neat/rules/transformers/_base.py +81 -0
- cognite/neat/rules/{models/information/_converter.py → transformers/_converters.py} +217 -21
- cognite/neat/rules/transformers/_map_onto.py +97 -0
- cognite/neat/rules/transformers/_pipelines.py +61 -0
- cognite/neat/rules/transformers/_verification.py +136 -0
- cognite/neat/{graph/stores → store}/_provenance.py +10 -1
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/cdf/data_classes.py +20 -0
- cognite/neat/utils/regex_patterns.py +6 -0
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +116 -47
- cognite/neat/workflows/steps/lib/current/rules_importer.py +30 -28
- cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/RECORD +105 -106
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/issues/errors/external.py +0 -21
- cognite/neat/issues/errors/properties.py +0 -75
- cognite/neat/issues/errors/resources.py +0 -123
- cognite/neat/issues/errors/schema.py +0 -0
- cognite/neat/issues/neat_warnings/__init__.py +0 -2
- cognite/neat/issues/neat_warnings/identifier.py +0 -27
- cognite/neat/issues/neat_warnings/models.py +0 -22
- cognite/neat/issues/neat_warnings/properties.py +0 -77
- cognite/neat/issues/neat_warnings/resources.py +0 -125
- cognite/neat/rules/issues/__init__.py +0 -22
- cognite/neat/rules/issues/base.py +0 -63
- cognite/neat/rules/issues/dms.py +0 -549
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/rules/models/_constants.py +0 -1
- cognite/neat/rules/models/_types/__init__.py +0 -19
- cognite/neat/rules/models/asset/_converter.py +0 -4
- cognite/neat/rules/models/dms/_converter.py +0 -145
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/entry_points.txt +0 -0
cognite/neat/rules/_shared.py
CHANGED
|
@@ -1,10 +1,61 @@
|
|
|
1
|
-
from
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any, Generic, TypeAlias, TypeVar
|
|
2
4
|
|
|
5
|
+
from cognite.neat.issues import IssueList
|
|
3
6
|
from cognite.neat.rules.models import (
|
|
4
7
|
AssetRules,
|
|
5
8
|
DMSRules,
|
|
6
9
|
DomainRules,
|
|
7
10
|
InformationRules,
|
|
8
11
|
)
|
|
12
|
+
from cognite.neat.rules.models.asset._rules_input import AssetInputRules
|
|
13
|
+
from cognite.neat.rules.models.dms._rules_input import DMSInputRules
|
|
14
|
+
from cognite.neat.rules.models.information._rules_input import InformationInputRules
|
|
9
15
|
|
|
10
|
-
|
|
16
|
+
VerifiedRules: TypeAlias = DomainRules | InformationRules | DMSRules | AssetRules
|
|
17
|
+
InputRules: TypeAlias = AssetInputRules | DMSInputRules | InformationInputRules
|
|
18
|
+
Rules: TypeAlias = (
|
|
19
|
+
AssetInputRules | DMSInputRules | InformationInputRules | DomainRules | InformationRules | DMSRules | AssetRules
|
|
20
|
+
)
|
|
21
|
+
T_Rules = TypeVar("T_Rules", bound=Rules)
|
|
22
|
+
T_VerifiedRules = TypeVar("T_VerifiedRules", bound=VerifiedRules)
|
|
23
|
+
T_InputRules = TypeVar("T_InputRules", bound=InputRules)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class OutRules(Generic[T_Rules], ABC):
|
|
28
|
+
"""This is a base class for all rule states."""
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def get_rules(self) -> T_Rules | None:
|
|
32
|
+
"""Get the rules from the state."""
|
|
33
|
+
raise NotImplementedError()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class JustRules(OutRules[T_Rules]):
|
|
38
|
+
"""This represents a rule that exists"""
|
|
39
|
+
|
|
40
|
+
rules: T_Rules
|
|
41
|
+
|
|
42
|
+
def get_rules(self) -> T_Rules:
|
|
43
|
+
return self.rules
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class MaybeRules(OutRules[T_Rules]):
|
|
48
|
+
"""This represents a rule that may or may not exist"""
|
|
49
|
+
|
|
50
|
+
rules: T_Rules | None
|
|
51
|
+
issues: IssueList
|
|
52
|
+
|
|
53
|
+
def get_rules(self) -> T_Rules | None:
|
|
54
|
+
return self.rules
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class ReadRules(MaybeRules[T_Rules]):
|
|
59
|
+
"""This represents a rule that does not exist"""
|
|
60
|
+
|
|
61
|
+
read_context: dict[str, Any]
|
|
@@ -9,7 +9,7 @@ from typing import Generic, TypeVar
|
|
|
9
9
|
import pandas as pd
|
|
10
10
|
from pydantic import BaseModel
|
|
11
11
|
|
|
12
|
-
from cognite.neat.rules.models.
|
|
12
|
+
from cognite.neat.rules.models._base_rules import BaseRules
|
|
13
13
|
from cognite.neat.rules.models._rdfpath import RDFPath
|
|
14
14
|
from cognite.neat.rules.models.entities import (
|
|
15
15
|
ClassEntity,
|
|
@@ -5,47 +5,36 @@ from typing import Generic, TypeVar
|
|
|
5
5
|
|
|
6
6
|
from cognite.client import CogniteClient
|
|
7
7
|
|
|
8
|
-
from cognite.neat.rules._shared import
|
|
9
|
-
from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
|
|
8
|
+
from cognite.neat.rules._shared import T_VerifiedRules
|
|
10
9
|
from cognite.neat.utils.auxiliary import class_html_doc
|
|
11
10
|
from cognite.neat.utils.upload import UploadResult, UploadResultList
|
|
12
11
|
|
|
13
12
|
T_Export = TypeVar("T_Export")
|
|
14
13
|
|
|
15
14
|
|
|
16
|
-
class BaseExporter(ABC, Generic[T_Export]):
|
|
15
|
+
class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
|
|
17
16
|
_new_line = "\n"
|
|
18
17
|
_encoding = "utf-8"
|
|
19
18
|
|
|
20
19
|
@abstractmethod
|
|
21
|
-
def export_to_file(self, rules:
|
|
20
|
+
def export_to_file(self, rules: T_VerifiedRules, filepath: Path) -> None:
|
|
22
21
|
raise NotImplementedError
|
|
23
22
|
|
|
24
23
|
@abstractmethod
|
|
25
|
-
def export(self, rules:
|
|
24
|
+
def export(self, rules: T_VerifiedRules) -> T_Export:
|
|
26
25
|
raise NotImplementedError
|
|
27
26
|
|
|
28
|
-
def _convert_to_output_role(self, rules: Rules, output_role: RoleTypes | None = None) -> Rules:
|
|
29
|
-
if rules.metadata.role is output_role or output_role is None:
|
|
30
|
-
return rules
|
|
31
|
-
elif output_role is RoleTypes.dms and isinstance(rules, InformationRules):
|
|
32
|
-
return rules.as_dms_rules()
|
|
33
|
-
elif output_role is RoleTypes.information and isinstance(rules, DMSRules):
|
|
34
|
-
return rules.as_information_rules()
|
|
35
|
-
else:
|
|
36
|
-
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
37
|
-
|
|
38
27
|
@classmethod
|
|
39
28
|
def _repr_html_(cls) -> str:
|
|
40
29
|
return class_html_doc(cls, include_factory_methods=False)
|
|
41
30
|
|
|
42
31
|
|
|
43
|
-
class CDFExporter(BaseExporter[T_Export]):
|
|
32
|
+
class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
|
|
44
33
|
@abstractmethod
|
|
45
34
|
def export_to_cdf_iterable(
|
|
46
|
-
self, rules:
|
|
35
|
+
self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False
|
|
47
36
|
) -> Iterable[UploadResult]:
|
|
48
37
|
raise NotImplementedError
|
|
49
38
|
|
|
50
|
-
def export_to_cdf(self, rules:
|
|
39
|
+
def export_to_cdf(self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
|
|
51
40
|
return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))
|
|
@@ -16,10 +16,10 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
16
16
|
from cognite.client.exceptions import CogniteAPIError
|
|
17
17
|
|
|
18
18
|
from cognite.neat.issues import IssueList
|
|
19
|
-
from cognite.neat.issues.
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
19
|
+
from cognite.neat.issues.warnings import (
|
|
20
|
+
PrincipleOneModelOneSpaceWarning,
|
|
21
|
+
ResourceRetrievalWarning,
|
|
22
|
+
)
|
|
23
23
|
from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
|
|
24
24
|
from cognite.neat.utils.cdf.loaders import (
|
|
25
25
|
ContainerLoader,
|
|
@@ -39,7 +39,7 @@ from ._base import CDFExporter
|
|
|
39
39
|
Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers", "node_types"]
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class DMSExporter(CDFExporter[DMSSchema]):
|
|
42
|
+
class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
|
|
43
43
|
"""Export rules to Cognite Data Fusion's Data Model Storage (DMS) service.
|
|
44
44
|
|
|
45
45
|
Args:
|
|
@@ -80,7 +80,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
80
80
|
self.suppress_warnings = suppress_warnings
|
|
81
81
|
self._schema: DMSSchema | None = None
|
|
82
82
|
|
|
83
|
-
def export_to_file(self, rules:
|
|
83
|
+
def export_to_file(self, rules: DMSRules, filepath: Path) -> None:
|
|
84
84
|
"""Export the rules to a file(s).
|
|
85
85
|
|
|
86
86
|
If the file is a directory, the components will be exported to separate files, otherwise they will be
|
|
@@ -95,12 +95,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
95
95
|
else:
|
|
96
96
|
self._export_to_zip_file(filepath, rules)
|
|
97
97
|
|
|
98
|
-
def _export_to_directory(self, directory: Path, rules:
|
|
98
|
+
def _export_to_directory(self, directory: Path, rules: DMSRules) -> None:
|
|
99
99
|
schema = self.export(rules)
|
|
100
100
|
exclude = self._create_exclude_set()
|
|
101
101
|
schema.to_directory(directory, exclude=exclude, new_line=self._new_line, encoding=self._encoding)
|
|
102
102
|
|
|
103
|
-
def _export_to_zip_file(self, filepath: Path, rules:
|
|
103
|
+
def _export_to_zip_file(self, filepath: Path, rules: DMSRules) -> None:
|
|
104
104
|
if filepath.suffix not in {".zip"}:
|
|
105
105
|
warnings.warn("File extension is not .zip, adding it to the file name", stacklevel=2)
|
|
106
106
|
filepath = filepath.with_suffix(".zip")
|
|
@@ -115,16 +115,10 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
115
115
|
exclude = {"spaces", "data_models", "views", "containers", "node_types"} - self.export_components
|
|
116
116
|
return exclude
|
|
117
117
|
|
|
118
|
-
def export(self, rules:
|
|
119
|
-
|
|
120
|
-
dms_rules = rules
|
|
121
|
-
elif isinstance(rules, InformationRules):
|
|
122
|
-
dms_rules = rules.as_dms_rules()
|
|
123
|
-
else:
|
|
124
|
-
raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
|
|
125
|
-
return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
|
|
118
|
+
def export(self, rules: DMSRules) -> DMSSchema:
|
|
119
|
+
return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
|
|
126
120
|
|
|
127
|
-
def delete_from_cdf(self, rules:
|
|
121
|
+
def delete_from_cdf(self, rules: DMSRules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
|
|
128
122
|
to_export = self._prepare_exporters(rules, client)
|
|
129
123
|
|
|
130
124
|
# we need to reverse order in which we are picking up the items to delete
|
|
@@ -168,7 +162,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
168
162
|
)
|
|
169
163
|
|
|
170
164
|
def export_to_cdf_iterable(
|
|
171
|
-
self, rules:
|
|
165
|
+
self, rules: DMSRules, client: CogniteClient, dry_run: bool = False
|
|
172
166
|
) -> Iterable[UploadResult]:
|
|
173
167
|
to_export = self._prepare_exporters(rules, client)
|
|
174
168
|
|
|
@@ -298,7 +292,10 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
298
292
|
if isinstance(loader, DataModelLoader):
|
|
299
293
|
models = cast(list[DataModelApply], items)
|
|
300
294
|
if other_models := self._exist_other_data_models(loader, models):
|
|
301
|
-
warning =
|
|
295
|
+
warning = PrincipleOneModelOneSpaceWarning(
|
|
296
|
+
f"There are multiple data models in the same space {models[0].space}. "
|
|
297
|
+
f"Other data models in the space are {other_models}.",
|
|
298
|
+
)
|
|
302
299
|
if not self.suppress_warnings:
|
|
303
300
|
warnings.warn(warning, stacklevel=2)
|
|
304
301
|
issue_list.append(warning)
|
|
@@ -314,7 +311,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
|
|
|
314
311
|
try:
|
|
315
312
|
data_models = loader.client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
|
|
316
313
|
except CogniteAPIError as e:
|
|
317
|
-
warnings.warn(
|
|
314
|
+
warnings.warn(ResourceRetrievalWarning(frozenset({space}), "space", str(e)), stacklevel=2)
|
|
318
315
|
return []
|
|
319
316
|
else:
|
|
320
317
|
return [
|
|
@@ -12,11 +12,10 @@ from openpyxl.cell import MergedCell
|
|
|
12
12
|
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
13
13
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
14
14
|
|
|
15
|
-
from cognite.neat.rules._shared import
|
|
15
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
16
16
|
from cognite.neat.rules.models import (
|
|
17
17
|
DataModelType,
|
|
18
18
|
ExtensionCategory,
|
|
19
|
-
RoleTypes,
|
|
20
19
|
SchemaCompleteness,
|
|
21
20
|
SheetEntity,
|
|
22
21
|
)
|
|
@@ -27,7 +26,7 @@ from cognite.neat.rules.models.information import InformationMetadata
|
|
|
27
26
|
from ._base import BaseExporter
|
|
28
27
|
|
|
29
28
|
|
|
30
|
-
class ExcelExporter(BaseExporter[Workbook]):
|
|
29
|
+
class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
31
30
|
"""Export rules to Excel.
|
|
32
31
|
|
|
33
32
|
Args:
|
|
@@ -68,16 +67,14 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
68
67
|
"Classes": "Definition of Classes",
|
|
69
68
|
"Views": "Definition of Views",
|
|
70
69
|
"Containers": "Definition of Containers",
|
|
70
|
+
"Nodes": "Definition of Nodes",
|
|
71
|
+
"Enum": "Definition of Enum Collections",
|
|
71
72
|
}
|
|
72
73
|
style_options = get_args(Style)
|
|
73
74
|
dump_options = get_args(DumpOptions)
|
|
74
75
|
|
|
75
76
|
def __init__(
|
|
76
|
-
self,
|
|
77
|
-
styling: Style = "default",
|
|
78
|
-
output_role: RoleTypes | None = None,
|
|
79
|
-
dump_as: DumpOptions = "user",
|
|
80
|
-
new_model_id: tuple[str, str] | None = None,
|
|
77
|
+
self, styling: Style = "default", dump_as: DumpOptions = "user", new_model_id: tuple[str, str] | None = None
|
|
81
78
|
):
|
|
82
79
|
if styling not in self.style_options:
|
|
83
80
|
raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
|
|
@@ -85,11 +82,10 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
85
82
|
raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
|
|
86
83
|
self.styling = styling
|
|
87
84
|
self._styling_level = self.style_options.index(styling)
|
|
88
|
-
self.output_role = output_role
|
|
89
85
|
self.new_model_id = new_model_id
|
|
90
86
|
self.dump_as = dump_as
|
|
91
87
|
|
|
92
|
-
def export_to_file(self, rules:
|
|
88
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
93
89
|
"""Exports transformation rules to excel file."""
|
|
94
90
|
data = self.export(rules)
|
|
95
91
|
try:
|
|
@@ -98,8 +94,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
98
94
|
data.close()
|
|
99
95
|
return None
|
|
100
96
|
|
|
101
|
-
def export(self, rules:
|
|
102
|
-
rules = self._convert_to_output_role(rules, self.output_role)
|
|
97
|
+
def export(self, rules: VerifiedRules) -> Workbook:
|
|
103
98
|
workbook = Workbook()
|
|
104
99
|
# Remove default sheet named "Sheet"
|
|
105
100
|
workbook.remove(workbook["Sheet"])
|
|
@@ -147,7 +142,7 @@ class ExcelExporter(BaseExporter[Workbook]):
|
|
|
147
142
|
self,
|
|
148
143
|
workbook: Workbook,
|
|
149
144
|
dumped_rules: dict[str, Any],
|
|
150
|
-
rules:
|
|
145
|
+
rules: VerifiedRules,
|
|
151
146
|
sheet_prefix: str = "",
|
|
152
147
|
):
|
|
153
148
|
for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
|
|
@@ -279,9 +274,7 @@ class _MetadataCreator:
|
|
|
279
274
|
|
|
280
275
|
new_metadata = self._create_new_info(now)
|
|
281
276
|
if isinstance(metadata, DMSMetadata):
|
|
282
|
-
from cognite.neat.rules.
|
|
283
|
-
_InformationRulesConverter,
|
|
284
|
-
)
|
|
277
|
+
from cognite.neat.rules.transformers._converters import _InformationRulesConverter
|
|
285
278
|
|
|
286
279
|
output_metadata: DMSMetadata | InformationMetadata = _InformationRulesConverter._convert_metadata_to_dms(
|
|
287
280
|
new_metadata
|
|
@@ -9,20 +9,12 @@ from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, BNode, Graph, Literal, Namespac
|
|
|
9
9
|
from rdflib.collection import Collection as GraphCollection
|
|
10
10
|
|
|
11
11
|
from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
|
|
12
|
-
from cognite.neat.
|
|
13
|
-
from cognite.neat.
|
|
14
|
-
|
|
15
|
-
MissingDataModelPrefixOrNamespaceWarning,
|
|
16
|
-
OntologyMultiDefinitionPropertyWarning,
|
|
17
|
-
OntologyMultiDomainPropertyWarning,
|
|
18
|
-
OntologyMultiLabeledPropertyWarning,
|
|
19
|
-
OntologyMultiRangePropertyWarning,
|
|
20
|
-
OntologyMultiTypePropertyWarning,
|
|
21
|
-
PrefixMissingError,
|
|
22
|
-
PropertiesDefinedMultipleTimesError,
|
|
23
|
-
PropertyDefinitionsNotForSamePropertyError,
|
|
12
|
+
from cognite.neat.issues import MultiValueError
|
|
13
|
+
from cognite.neat.issues.errors import (
|
|
14
|
+
PropertyDefinitionDuplicatedError,
|
|
24
15
|
)
|
|
25
|
-
from cognite.neat.
|
|
16
|
+
from cognite.neat.issues.warnings import PropertyDefinitionDuplicatedWarning
|
|
17
|
+
from cognite.neat.rules.analysis import InformationAnalysis
|
|
26
18
|
from cognite.neat.rules.models.data_types import DataType
|
|
27
19
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
|
|
28
20
|
from cognite.neat.rules.models.information import (
|
|
@@ -31,43 +23,40 @@ from cognite.neat.rules.models.information import (
|
|
|
31
23
|
InformationProperty,
|
|
32
24
|
InformationRules,
|
|
33
25
|
)
|
|
34
|
-
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
35
26
|
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
36
27
|
|
|
37
28
|
from ._base import BaseExporter
|
|
38
|
-
from ._validation import
|
|
29
|
+
from ._validation import duplicated_properties
|
|
39
30
|
|
|
40
31
|
if sys.version_info >= (3, 11):
|
|
41
32
|
from typing import Self
|
|
42
33
|
else:
|
|
43
34
|
from typing_extensions import Self
|
|
44
35
|
|
|
45
|
-
from cognite.neat.rules._shared import Rules
|
|
46
36
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def export_to_file(self, rules: Rules, filepath: Path) -> None:
|
|
37
|
+
class GraphExporter(BaseExporter[InformationRules, Graph], ABC):
|
|
38
|
+
def export_to_file(self, rules: InformationRules, filepath: Path) -> None:
|
|
50
39
|
self.export(rules).serialize(destination=filepath, encoding=self._encoding, newline=self._new_line)
|
|
51
40
|
|
|
52
41
|
|
|
53
42
|
class OWLExporter(GraphExporter):
|
|
54
|
-
"""Exports rules to an OWL ontology."""
|
|
43
|
+
"""Exports verified information rules to an OWL ontology."""
|
|
55
44
|
|
|
56
|
-
def export(self, rules:
|
|
45
|
+
def export(self, rules: InformationRules) -> Graph:
|
|
57
46
|
return Ontology.from_rules(rules).as_owl()
|
|
58
47
|
|
|
59
48
|
|
|
60
49
|
class SHACLExporter(GraphExporter):
|
|
61
50
|
"""Exports rules to a SHACL graph."""
|
|
62
51
|
|
|
63
|
-
def export(self, rules:
|
|
52
|
+
def export(self, rules: InformationRules) -> Graph:
|
|
64
53
|
return Ontology.from_rules(rules).as_shacl()
|
|
65
54
|
|
|
66
55
|
|
|
67
56
|
class SemanticDataModelExporter(GraphExporter):
|
|
68
|
-
"""Exports rules to a semantic data model."""
|
|
57
|
+
"""Exports verified information rules to a semantic data model."""
|
|
69
58
|
|
|
70
|
-
def export(self, rules:
|
|
59
|
+
def export(self, rules: InformationRules) -> Graph:
|
|
71
60
|
return Ontology.from_rules(rules).as_semantic_data_model()
|
|
72
61
|
|
|
73
62
|
|
|
@@ -94,34 +83,30 @@ class Ontology(OntologyModel):
|
|
|
94
83
|
prefixes: dict[str, Namespace]
|
|
95
84
|
|
|
96
85
|
@classmethod
|
|
97
|
-
def from_rules(cls,
|
|
86
|
+
def from_rules(cls, rules: InformationRules) -> Self:
|
|
98
87
|
"""
|
|
99
88
|
Generates an ontology from a set of transformation rules.
|
|
100
89
|
|
|
101
90
|
Args:
|
|
102
|
-
|
|
91
|
+
rules: The rules to generate the ontology from.
|
|
103
92
|
|
|
104
93
|
Returns:
|
|
105
94
|
An instance of Ontology.
|
|
106
95
|
"""
|
|
107
|
-
if
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
raise PrefixMissingError().as_exception()
|
|
122
|
-
|
|
123
|
-
if rules.metadata.namespace is None:
|
|
124
|
-
raise MissingDataModelPrefixOrNamespaceWarning()
|
|
96
|
+
if duplicates := duplicated_properties(rules.properties):
|
|
97
|
+
errors = []
|
|
98
|
+
for (class_, property_), definitions in duplicates.items():
|
|
99
|
+
errors.append(
|
|
100
|
+
PropertyDefinitionDuplicatedError(
|
|
101
|
+
class_,
|
|
102
|
+
"class",
|
|
103
|
+
property_,
|
|
104
|
+
frozenset({str(definition[1].value_type) for definition in definitions}),
|
|
105
|
+
tuple(definition[0] for definition in definitions),
|
|
106
|
+
"rows",
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
raise MultiValueError(errors)
|
|
125
110
|
|
|
126
111
|
class_dict = InformationAnalysis(rules).as_class_dict()
|
|
127
112
|
return cls(
|
|
@@ -184,9 +169,6 @@ class Ontology(OntologyModel):
|
|
|
184
169
|
for prefix, namespace in self.prefixes.items():
|
|
185
170
|
owl.bind(prefix, namespace)
|
|
186
171
|
|
|
187
|
-
if self.metadata.namespace is None:
|
|
188
|
-
raise MetadataSheetNamespaceNotDefinedError().as_exception()
|
|
189
|
-
|
|
190
172
|
owl.add((URIRef(self.metadata.namespace), RDF.type, OWL.Ontology))
|
|
191
173
|
for property_ in self.properties:
|
|
192
174
|
for triple in property_.triples:
|
|
@@ -233,8 +215,6 @@ class OWLMetadata(InformationMetadata):
|
|
|
233
215
|
@property
|
|
234
216
|
def triples(self) -> list[tuple]:
|
|
235
217
|
# Mandatory triples originating from Metadata mandatory fields
|
|
236
|
-
if self.namespace is None:
|
|
237
|
-
raise MetadataSheetNamespaceNotDefinedError().as_exception()
|
|
238
218
|
triples: list[tuple] = [
|
|
239
219
|
(URIRef(self.namespace), DCTERMS.hasVersion, Literal(self.version)),
|
|
240
220
|
(URIRef(self.namespace), OWL.versionInfo, Literal(self.version)),
|
|
@@ -323,16 +303,17 @@ class OWLProperty(OntologyModel):
|
|
|
323
303
|
range_: set[URIRef]
|
|
324
304
|
namespace: Namespace
|
|
325
305
|
|
|
326
|
-
@staticmethod
|
|
327
|
-
def same_property_id(definitions: list[InformationProperty]) -> bool:
|
|
328
|
-
return len({definition.property_ for definition in definitions}) == 1
|
|
329
|
-
|
|
330
306
|
@classmethod
|
|
331
307
|
def from_list_of_properties(cls, definitions: list[InformationProperty], namespace: Namespace) -> "OWLProperty":
|
|
332
308
|
"""Here list of properties is a list of properties with the same id, but different definitions."""
|
|
333
|
-
|
|
334
|
-
if
|
|
335
|
-
raise
|
|
309
|
+
property_ids = {definition.property_ for definition in definitions}
|
|
310
|
+
if len(property_ids) != 1:
|
|
311
|
+
raise PropertyDefinitionDuplicatedError(
|
|
312
|
+
definitions[0].class_,
|
|
313
|
+
"class",
|
|
314
|
+
definitions[0].property_,
|
|
315
|
+
frozenset(property_ids),
|
|
316
|
+
)
|
|
336
317
|
|
|
337
318
|
owl_property = cls.model_construct(
|
|
338
319
|
id_=namespace[definitions[0].property_],
|
|
@@ -363,8 +344,15 @@ class OWLProperty(OntologyModel):
|
|
|
363
344
|
def is_multi_type(cls, v, info: ValidationInfo):
|
|
364
345
|
if len(v) > 1:
|
|
365
346
|
warnings.warn(
|
|
366
|
-
|
|
367
|
-
remove_namespace_from_uri(info.data["
|
|
347
|
+
PropertyDefinitionDuplicatedWarning(
|
|
348
|
+
remove_namespace_from_uri(info.data["id"]),
|
|
349
|
+
"class",
|
|
350
|
+
"type",
|
|
351
|
+
frozenset({remove_namespace_from_uri(t) for t in v}),
|
|
352
|
+
"This warning occurs when a same property is define for two object/classes where"
|
|
353
|
+
" its expected value type is different in one definition, e.g. acts as an edge, while in "
|
|
354
|
+
"other definition acts as and attribute",
|
|
355
|
+
"If a property takes different value types for different objects, simply define new property",
|
|
368
356
|
),
|
|
369
357
|
stacklevel=2,
|
|
370
358
|
)
|
|
@@ -374,8 +362,14 @@ class OWLProperty(OntologyModel):
|
|
|
374
362
|
def is_multi_range(cls, v, info: ValidationInfo):
|
|
375
363
|
if len(v) > 1:
|
|
376
364
|
warnings.warn(
|
|
377
|
-
|
|
378
|
-
remove_namespace_from_uri(info.data["id_"]),
|
|
365
|
+
PropertyDefinitionDuplicatedWarning(
|
|
366
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
367
|
+
"class",
|
|
368
|
+
"range",
|
|
369
|
+
frozenset({remove_namespace_from_uri(t) for t in v}),
|
|
370
|
+
"This warning occurs when a property takes range of "
|
|
371
|
+
"values which consists of union of multiple value types.",
|
|
372
|
+
"If value types for different objects, simply define new property",
|
|
379
373
|
),
|
|
380
374
|
stacklevel=2,
|
|
381
375
|
)
|
|
@@ -385,8 +379,15 @@ class OWLProperty(OntologyModel):
|
|
|
385
379
|
def is_multi_domain(cls, v, info: ValidationInfo):
|
|
386
380
|
if len(v) > 1:
|
|
387
381
|
warnings.warn(
|
|
388
|
-
|
|
389
|
-
remove_namespace_from_uri(info.data["id_"]),
|
|
382
|
+
PropertyDefinitionDuplicatedWarning(
|
|
383
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
384
|
+
"class",
|
|
385
|
+
"domain",
|
|
386
|
+
frozenset({remove_namespace_from_uri(t) for t in v}),
|
|
387
|
+
"This warning occurs when a same property is define for two object/classes where"
|
|
388
|
+
" its expected value type is different in one definition, e.g. acts as an edge, while in "
|
|
389
|
+
"other definition acts as and attribute",
|
|
390
|
+
"If value types for different objects, simply define new property",
|
|
390
391
|
),
|
|
391
392
|
stacklevel=2,
|
|
392
393
|
)
|
|
@@ -396,7 +397,13 @@ class OWLProperty(OntologyModel):
|
|
|
396
397
|
def has_multi_name(cls, v, info: ValidationInfo):
|
|
397
398
|
if len(v) > 1:
|
|
398
399
|
warnings.warn(
|
|
399
|
-
|
|
400
|
+
PropertyDefinitionDuplicatedWarning(
|
|
401
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
402
|
+
"class",
|
|
403
|
+
"label",
|
|
404
|
+
frozenset(v),
|
|
405
|
+
f"Only the first label (name) will be used, {v[0]}",
|
|
406
|
+
),
|
|
400
407
|
stacklevel=2,
|
|
401
408
|
)
|
|
402
409
|
return v
|
|
@@ -405,7 +412,13 @@ class OWLProperty(OntologyModel):
|
|
|
405
412
|
def has_multi_comment(cls, v, info: ValidationInfo):
|
|
406
413
|
if len(v) > 1:
|
|
407
414
|
warnings.warn(
|
|
408
|
-
|
|
415
|
+
PropertyDefinitionDuplicatedWarning(
|
|
416
|
+
remove_namespace_from_uri(info.data["id_"]),
|
|
417
|
+
"class",
|
|
418
|
+
"comment",
|
|
419
|
+
frozenset(v),
|
|
420
|
+
"All definitions will be concatenated to form a single definition.",
|
|
421
|
+
),
|
|
409
422
|
stacklevel=2,
|
|
410
423
|
)
|
|
411
424
|
return v
|
|
@@ -5,14 +5,13 @@ from typing import Literal, get_args
|
|
|
5
5
|
|
|
6
6
|
import yaml
|
|
7
7
|
|
|
8
|
-
from cognite.neat.rules._shared import
|
|
9
|
-
from cognite.neat.rules.models import RoleTypes
|
|
8
|
+
from cognite.neat.rules._shared import VerifiedRules
|
|
10
9
|
|
|
11
10
|
from ._base import BaseExporter
|
|
12
11
|
|
|
13
12
|
|
|
14
|
-
class YAMLExporter(BaseExporter[str]):
|
|
15
|
-
"""Export rules to YAML.
|
|
13
|
+
class YAMLExporter(BaseExporter[VerifiedRules, str]):
|
|
14
|
+
"""Export rules (Information, DMS or Domain) to YAML.
|
|
16
15
|
|
|
17
16
|
Args:
|
|
18
17
|
files: The number of files to output. Defaults to "single".
|
|
@@ -38,16 +37,15 @@ class YAMLExporter(BaseExporter[str]):
|
|
|
38
37
|
file_option = get_args(Files)
|
|
39
38
|
format_option = get_args(Format)
|
|
40
39
|
|
|
41
|
-
def __init__(self, files: Files = "single", output: Format = "yaml"
|
|
40
|
+
def __init__(self, files: Files = "single", output: Format = "yaml"):
|
|
42
41
|
if files not in self.file_option:
|
|
43
42
|
raise ValueError(f"Invalid files: {files}. Valid options are {self.file_option}")
|
|
44
43
|
if output not in self.format_option:
|
|
45
44
|
raise ValueError(f"Invalid output: {output}. Valid options are {self.format_option}")
|
|
46
45
|
self.files = files
|
|
47
46
|
self.output = output
|
|
48
|
-
self.output_role = output_role
|
|
49
47
|
|
|
50
|
-
def export_to_file(self, rules:
|
|
48
|
+
def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
|
|
51
49
|
"""Exports transformation rules to YAML/JSON file(s)."""
|
|
52
50
|
if self.files == "single":
|
|
53
51
|
if filepath.suffix != f".{self.output}":
|
|
@@ -57,7 +55,7 @@ class YAMLExporter(BaseExporter[str]):
|
|
|
57
55
|
else:
|
|
58
56
|
raise NotImplementedError(f"Exporting to {self.files} files is not supported")
|
|
59
57
|
|
|
60
|
-
def export(self, rules:
|
|
58
|
+
def export(self, rules: VerifiedRules) -> str:
|
|
61
59
|
"""Export rules to YAML (or JSON) format.
|
|
62
60
|
|
|
63
61
|
Args:
|
|
@@ -66,7 +64,6 @@ class YAMLExporter(BaseExporter[str]):
|
|
|
66
64
|
Returns:
|
|
67
65
|
str: The rules in YAML (or JSON) format.
|
|
68
66
|
"""
|
|
69
|
-
rules = self._convert_to_output_role(rules, self.output_role)
|
|
70
67
|
# model_dump_json ensures that the output is in JSON format,
|
|
71
68
|
# if we don't do this, we will get Enums and other types that are not serializable to YAML
|
|
72
69
|
json_output = rules.dump(mode="json")
|