cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (129) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/constants.py +3 -0
  3. cognite/neat/graph/__init__.py +0 -3
  4. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -1
  5. cognite/neat/graph/loaders/_base.py +3 -3
  6. cognite/neat/graph/loaders/_rdf2asset.py +24 -25
  7. cognite/neat/graph/loaders/_rdf2dms.py +20 -15
  8. cognite/neat/issues/__init__.py +1 -3
  9. cognite/neat/issues/_base.py +261 -71
  10. cognite/neat/issues/errors/__init__.py +73 -0
  11. cognite/neat/issues/errors/_external.py +67 -0
  12. cognite/neat/issues/errors/_general.py +35 -0
  13. cognite/neat/issues/errors/_properties.py +62 -0
  14. cognite/neat/issues/errors/_resources.py +111 -0
  15. cognite/neat/issues/errors/_workflow.py +36 -0
  16. cognite/neat/issues/formatters.py +1 -1
  17. cognite/neat/issues/warnings/__init__.py +66 -0
  18. cognite/neat/issues/warnings/_external.py +40 -0
  19. cognite/neat/issues/warnings/_general.py +29 -0
  20. cognite/neat/issues/warnings/_models.py +92 -0
  21. cognite/neat/issues/warnings/_properties.py +44 -0
  22. cognite/neat/issues/warnings/_resources.py +55 -0
  23. cognite/neat/issues/warnings/user_modeling.py +113 -0
  24. cognite/neat/rules/_shared.py +53 -2
  25. cognite/neat/rules/analysis/_base.py +1 -1
  26. cognite/neat/rules/exporters/_base.py +7 -18
  27. cognite/neat/rules/exporters/_rules2dms.py +17 -20
  28. cognite/neat/rules/exporters/_rules2excel.py +9 -16
  29. cognite/neat/rules/exporters/_rules2ontology.py +77 -64
  30. cognite/neat/rules/exporters/_rules2yaml.py +6 -9
  31. cognite/neat/rules/exporters/_validation.py +11 -96
  32. cognite/neat/rules/importers/_base.py +9 -58
  33. cognite/neat/rules/importers/_dms2rules.py +188 -135
  34. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +48 -35
  35. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +36 -45
  36. cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
  37. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +8 -4
  38. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +3 -3
  39. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +18 -11
  40. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +12 -19
  41. cognite/neat/rules/importers/_rdf/_inference2rules.py +14 -37
  42. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
  43. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
  44. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +9 -20
  45. cognite/neat/rules/importers/_rdf/_shared.py +4 -4
  46. cognite/neat/rules/importers/_spreadsheet2rules.py +46 -97
  47. cognite/neat/rules/importers/_yaml2rules.py +32 -58
  48. cognite/neat/rules/models/__init__.py +21 -5
  49. cognite/neat/rules/models/_base_input.py +162 -0
  50. cognite/neat/rules/models/{_base.py → _base_rules.py} +1 -12
  51. cognite/neat/rules/models/_rdfpath.py +4 -4
  52. cognite/neat/rules/models/{_types/_field.py → _types.py} +5 -10
  53. cognite/neat/rules/models/asset/__init__.py +5 -2
  54. cognite/neat/rules/models/asset/_rules.py +3 -23
  55. cognite/neat/rules/models/asset/_rules_input.py +40 -115
  56. cognite/neat/rules/models/asset/_validation.py +14 -10
  57. cognite/neat/rules/models/data_types.py +150 -44
  58. cognite/neat/rules/models/dms/__init__.py +19 -7
  59. cognite/neat/rules/models/dms/_exporter.py +102 -34
  60. cognite/neat/rules/models/dms/_rules.py +65 -162
  61. cognite/neat/rules/models/dms/_rules_input.py +186 -254
  62. cognite/neat/rules/models/dms/_schema.py +87 -78
  63. cognite/neat/rules/models/dms/_serializer.py +44 -3
  64. cognite/neat/rules/models/dms/_validation.py +106 -68
  65. cognite/neat/rules/models/domain.py +52 -1
  66. cognite/neat/rules/models/entities/__init__.py +63 -0
  67. cognite/neat/rules/models/entities/_constants.py +73 -0
  68. cognite/neat/rules/models/entities/_loaders.py +76 -0
  69. cognite/neat/rules/models/entities/_multi_value.py +67 -0
  70. cognite/neat/rules/models/{entities.py → entities/_single_value.py} +74 -232
  71. cognite/neat/rules/models/entities/_types.py +86 -0
  72. cognite/neat/rules/models/{wrapped_entities.py → entities/_wrapped.py} +1 -1
  73. cognite/neat/rules/models/information/__init__.py +10 -2
  74. cognite/neat/rules/models/information/_rules.py +10 -22
  75. cognite/neat/rules/models/information/_rules_input.py +57 -204
  76. cognite/neat/rules/models/information/_validation.py +48 -25
  77. cognite/neat/rules/transformers/__init__.py +21 -0
  78. cognite/neat/rules/transformers/_base.py +81 -0
  79. cognite/neat/rules/{models/information/_converter.py → transformers/_converters.py} +217 -21
  80. cognite/neat/rules/transformers/_map_onto.py +97 -0
  81. cognite/neat/rules/transformers/_pipelines.py +61 -0
  82. cognite/neat/rules/transformers/_verification.py +136 -0
  83. cognite/neat/{graph/stores → store}/_provenance.py +10 -1
  84. cognite/neat/utils/auxiliary.py +2 -35
  85. cognite/neat/utils/cdf/data_classes.py +20 -0
  86. cognite/neat/utils/regex_patterns.py +6 -0
  87. cognite/neat/utils/text.py +17 -0
  88. cognite/neat/workflows/base.py +4 -4
  89. cognite/neat/workflows/cdf_store.py +3 -3
  90. cognite/neat/workflows/steps/data_contracts.py +1 -1
  91. cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
  92. cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
  93. cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
  94. cognite/neat/workflows/steps/lib/current/rules_exporter.py +116 -47
  95. cognite/neat/workflows/steps/lib/current/rules_importer.py +30 -28
  96. cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
  97. cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
  98. cognite/neat/workflows/steps_registry.py +4 -5
  99. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/METADATA +1 -1
  100. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/RECORD +105 -106
  101. cognite/neat/exceptions.py +0 -145
  102. cognite/neat/graph/exceptions.py +0 -90
  103. cognite/neat/issues/errors/external.py +0 -21
  104. cognite/neat/issues/errors/properties.py +0 -75
  105. cognite/neat/issues/errors/resources.py +0 -123
  106. cognite/neat/issues/errors/schema.py +0 -0
  107. cognite/neat/issues/neat_warnings/__init__.py +0 -2
  108. cognite/neat/issues/neat_warnings/identifier.py +0 -27
  109. cognite/neat/issues/neat_warnings/models.py +0 -22
  110. cognite/neat/issues/neat_warnings/properties.py +0 -77
  111. cognite/neat/issues/neat_warnings/resources.py +0 -125
  112. cognite/neat/rules/issues/__init__.py +0 -22
  113. cognite/neat/rules/issues/base.py +0 -63
  114. cognite/neat/rules/issues/dms.py +0 -549
  115. cognite/neat/rules/issues/fileread.py +0 -197
  116. cognite/neat/rules/issues/ontology.py +0 -298
  117. cognite/neat/rules/issues/spreadsheet.py +0 -563
  118. cognite/neat/rules/issues/spreadsheet_file.py +0 -151
  119. cognite/neat/rules/issues/tables.py +0 -72
  120. cognite/neat/rules/models/_constants.py +0 -1
  121. cognite/neat/rules/models/_types/__init__.py +0 -19
  122. cognite/neat/rules/models/asset/_converter.py +0 -4
  123. cognite/neat/rules/models/dms/_converter.py +0 -145
  124. cognite/neat/workflows/_exceptions.py +0 -41
  125. /cognite/neat/{graph/stores → store}/__init__.py +0 -0
  126. /cognite/neat/{graph/stores → store}/_base.py +0 -0
  127. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/LICENSE +0 -0
  128. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/WHEEL +0 -0
  129. {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/entry_points.txt +0 -0
@@ -1,10 +1,61 @@
1
- from typing import TypeAlias
1
+ from abc import ABC, abstractmethod
2
+ from dataclasses import dataclass
3
+ from typing import Any, Generic, TypeAlias, TypeVar
2
4
 
5
+ from cognite.neat.issues import IssueList
3
6
  from cognite.neat.rules.models import (
4
7
  AssetRules,
5
8
  DMSRules,
6
9
  DomainRules,
7
10
  InformationRules,
8
11
  )
12
+ from cognite.neat.rules.models.asset._rules_input import AssetInputRules
13
+ from cognite.neat.rules.models.dms._rules_input import DMSInputRules
14
+ from cognite.neat.rules.models.information._rules_input import InformationInputRules
9
15
 
10
- Rules: TypeAlias = DomainRules | InformationRules | DMSRules | AssetRules
16
+ VerifiedRules: TypeAlias = DomainRules | InformationRules | DMSRules | AssetRules
17
+ InputRules: TypeAlias = AssetInputRules | DMSInputRules | InformationInputRules
18
+ Rules: TypeAlias = (
19
+ AssetInputRules | DMSInputRules | InformationInputRules | DomainRules | InformationRules | DMSRules | AssetRules
20
+ )
21
+ T_Rules = TypeVar("T_Rules", bound=Rules)
22
+ T_VerifiedRules = TypeVar("T_VerifiedRules", bound=VerifiedRules)
23
+ T_InputRules = TypeVar("T_InputRules", bound=InputRules)
24
+
25
+
26
+ @dataclass
27
+ class OutRules(Generic[T_Rules], ABC):
28
+ """This is a base class for all rule states."""
29
+
30
+ @abstractmethod
31
+ def get_rules(self) -> T_Rules | None:
32
+ """Get the rules from the state."""
33
+ raise NotImplementedError()
34
+
35
+
36
+ @dataclass
37
+ class JustRules(OutRules[T_Rules]):
38
+ """This represents a rule that exists"""
39
+
40
+ rules: T_Rules
41
+
42
+ def get_rules(self) -> T_Rules:
43
+ return self.rules
44
+
45
+
46
+ @dataclass
47
+ class MaybeRules(OutRules[T_Rules]):
48
+ """This represents a rule that may or may not exist"""
49
+
50
+ rules: T_Rules | None
51
+ issues: IssueList
52
+
53
+ def get_rules(self) -> T_Rules | None:
54
+ return self.rules
55
+
56
+
57
+ @dataclass
58
+ class ReadRules(MaybeRules[T_Rules]):
59
+ """This represents a rule that does not exist"""
60
+
61
+ read_context: dict[str, Any]
@@ -9,7 +9,7 @@ from typing import Generic, TypeVar
9
9
  import pandas as pd
10
10
  from pydantic import BaseModel
11
11
 
12
- from cognite.neat.rules.models._base import BaseRules
12
+ from cognite.neat.rules.models._base_rules import BaseRules
13
13
  from cognite.neat.rules.models._rdfpath import RDFPath
14
14
  from cognite.neat.rules.models.entities import (
15
15
  ClassEntity,
@@ -5,47 +5,36 @@ from typing import Generic, TypeVar
5
5
 
6
6
  from cognite.client import CogniteClient
7
7
 
8
- from cognite.neat.rules._shared import Rules
9
- from cognite.neat.rules.models import DMSRules, InformationRules, RoleTypes
8
+ from cognite.neat.rules._shared import T_VerifiedRules
10
9
  from cognite.neat.utils.auxiliary import class_html_doc
11
10
  from cognite.neat.utils.upload import UploadResult, UploadResultList
12
11
 
13
12
  T_Export = TypeVar("T_Export")
14
13
 
15
14
 
16
- class BaseExporter(ABC, Generic[T_Export]):
15
+ class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
17
16
  _new_line = "\n"
18
17
  _encoding = "utf-8"
19
18
 
20
19
  @abstractmethod
21
- def export_to_file(self, rules: Rules, filepath: Path) -> None:
20
+ def export_to_file(self, rules: T_VerifiedRules, filepath: Path) -> None:
22
21
  raise NotImplementedError
23
22
 
24
23
  @abstractmethod
25
- def export(self, rules: Rules) -> T_Export:
24
+ def export(self, rules: T_VerifiedRules) -> T_Export:
26
25
  raise NotImplementedError
27
26
 
28
- def _convert_to_output_role(self, rules: Rules, output_role: RoleTypes | None = None) -> Rules:
29
- if rules.metadata.role is output_role or output_role is None:
30
- return rules
31
- elif output_role is RoleTypes.dms and isinstance(rules, InformationRules):
32
- return rules.as_dms_rules()
33
- elif output_role is RoleTypes.information and isinstance(rules, DMSRules):
34
- return rules.as_information_rules()
35
- else:
36
- raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
37
-
38
27
  @classmethod
39
28
  def _repr_html_(cls) -> str:
40
29
  return class_html_doc(cls, include_factory_methods=False)
41
30
 
42
31
 
43
- class CDFExporter(BaseExporter[T_Export]):
32
+ class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
44
33
  @abstractmethod
45
34
  def export_to_cdf_iterable(
46
- self, rules: Rules, client: CogniteClient, dry_run: bool = False
35
+ self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False
47
36
  ) -> Iterable[UploadResult]:
48
37
  raise NotImplementedError
49
38
 
50
- def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
39
+ def export_to_cdf(self, rules: T_VerifiedRules, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
51
40
  return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run))
@@ -16,10 +16,10 @@ from cognite.client.data_classes.data_modeling import (
16
16
  from cognite.client.exceptions import CogniteAPIError
17
17
 
18
18
  from cognite.neat.issues import IssueList
19
- from cognite.neat.issues.neat_warnings.resources import FailedLoadingResourcesWarning
20
- from cognite.neat.rules import issues
21
- from cognite.neat.rules._shared import Rules
22
- from cognite.neat.rules.models import InformationRules
19
+ from cognite.neat.issues.warnings import (
20
+ PrincipleOneModelOneSpaceWarning,
21
+ ResourceRetrievalWarning,
22
+ )
23
23
  from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
24
24
  from cognite.neat.utils.cdf.loaders import (
25
25
  ContainerLoader,
@@ -39,7 +39,7 @@ from ._base import CDFExporter
39
39
  Component: TypeAlias = Literal["all", "spaces", "data_models", "views", "containers", "node_types"]
40
40
 
41
41
 
42
- class DMSExporter(CDFExporter[DMSSchema]):
42
+ class DMSExporter(CDFExporter[DMSRules, DMSSchema]):
43
43
  """Export rules to Cognite Data Fusion's Data Model Storage (DMS) service.
44
44
 
45
45
  Args:
@@ -80,7 +80,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
80
80
  self.suppress_warnings = suppress_warnings
81
81
  self._schema: DMSSchema | None = None
82
82
 
83
- def export_to_file(self, rules: Rules, filepath: Path) -> None:
83
+ def export_to_file(self, rules: DMSRules, filepath: Path) -> None:
84
84
  """Export the rules to a file(s).
85
85
 
86
86
  If the file is a directory, the components will be exported to separate files, otherwise they will be
@@ -95,12 +95,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
95
95
  else:
96
96
  self._export_to_zip_file(filepath, rules)
97
97
 
98
- def _export_to_directory(self, directory: Path, rules: Rules) -> None:
98
+ def _export_to_directory(self, directory: Path, rules: DMSRules) -> None:
99
99
  schema = self.export(rules)
100
100
  exclude = self._create_exclude_set()
101
101
  schema.to_directory(directory, exclude=exclude, new_line=self._new_line, encoding=self._encoding)
102
102
 
103
- def _export_to_zip_file(self, filepath: Path, rules: Rules) -> None:
103
+ def _export_to_zip_file(self, filepath: Path, rules: DMSRules) -> None:
104
104
  if filepath.suffix not in {".zip"}:
105
105
  warnings.warn("File extension is not .zip, adding it to the file name", stacklevel=2)
106
106
  filepath = filepath.with_suffix(".zip")
@@ -115,16 +115,10 @@ class DMSExporter(CDFExporter[DMSSchema]):
115
115
  exclude = {"spaces", "data_models", "views", "containers", "node_types"} - self.export_components
116
116
  return exclude
117
117
 
118
- def export(self, rules: Rules) -> DMSSchema:
119
- if isinstance(rules, DMSRules):
120
- dms_rules = rules
121
- elif isinstance(rules, InformationRules):
122
- dms_rules = rules.as_dms_rules()
123
- else:
124
- raise ValueError(f"{type(rules).__name__} cannot be exported to DMS")
125
- return dms_rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
118
+ def export(self, rules: DMSRules) -> DMSSchema:
119
+ return rules.as_schema(include_pipeline=self.export_pipeline, instance_space=self.instance_space)
126
120
 
127
- def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
121
+ def delete_from_cdf(self, rules: DMSRules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
128
122
  to_export = self._prepare_exporters(rules, client)
129
123
 
130
124
  # we need to reverse order in which we are picking up the items to delete
@@ -168,7 +162,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
168
162
  )
169
163
 
170
164
  def export_to_cdf_iterable(
171
- self, rules: Rules, client: CogniteClient, dry_run: bool = False
165
+ self, rules: DMSRules, client: CogniteClient, dry_run: bool = False
172
166
  ) -> Iterable[UploadResult]:
173
167
  to_export = self._prepare_exporters(rules, client)
174
168
 
@@ -298,7 +292,10 @@ class DMSExporter(CDFExporter[DMSSchema]):
298
292
  if isinstance(loader, DataModelLoader):
299
293
  models = cast(list[DataModelApply], items)
300
294
  if other_models := self._exist_other_data_models(loader, models):
301
- warning = issues.dms.OtherDataModelsInSpaceWarning(models[0].space, other_models)
295
+ warning = PrincipleOneModelOneSpaceWarning(
296
+ f"There are multiple data models in the same space {models[0].space}. "
297
+ f"Other data models in the space are {other_models}.",
298
+ )
302
299
  if not self.suppress_warnings:
303
300
  warnings.warn(warning, stacklevel=2)
304
301
  issue_list.append(warning)
@@ -314,7 +311,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
314
311
  try:
315
312
  data_models = loader.client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
316
313
  except CogniteAPIError as e:
317
- warnings.warn(FailedLoadingResourcesWarning[str](frozenset({space}), "Space", str(e)), stacklevel=2)
314
+ warnings.warn(ResourceRetrievalWarning(frozenset({space}), "space", str(e)), stacklevel=2)
318
315
  return []
319
316
  else:
320
317
  return [
@@ -12,11 +12,10 @@ from openpyxl.cell import MergedCell
12
12
  from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
13
13
  from openpyxl.worksheet.worksheet import Worksheet
14
14
 
15
- from cognite.neat.rules._shared import Rules
15
+ from cognite.neat.rules._shared import VerifiedRules
16
16
  from cognite.neat.rules.models import (
17
17
  DataModelType,
18
18
  ExtensionCategory,
19
- RoleTypes,
20
19
  SchemaCompleteness,
21
20
  SheetEntity,
22
21
  )
@@ -27,7 +26,7 @@ from cognite.neat.rules.models.information import InformationMetadata
27
26
  from ._base import BaseExporter
28
27
 
29
28
 
30
- class ExcelExporter(BaseExporter[Workbook]):
29
+ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
31
30
  """Export rules to Excel.
32
31
 
33
32
  Args:
@@ -68,16 +67,14 @@ class ExcelExporter(BaseExporter[Workbook]):
68
67
  "Classes": "Definition of Classes",
69
68
  "Views": "Definition of Views",
70
69
  "Containers": "Definition of Containers",
70
+ "Nodes": "Definition of Nodes",
71
+ "Enum": "Definition of Enum Collections",
71
72
  }
72
73
  style_options = get_args(Style)
73
74
  dump_options = get_args(DumpOptions)
74
75
 
75
76
  def __init__(
76
- self,
77
- styling: Style = "default",
78
- output_role: RoleTypes | None = None,
79
- dump_as: DumpOptions = "user",
80
- new_model_id: tuple[str, str] | None = None,
77
+ self, styling: Style = "default", dump_as: DumpOptions = "user", new_model_id: tuple[str, str] | None = None
81
78
  ):
82
79
  if styling not in self.style_options:
83
80
  raise ValueError(f"Invalid styling: {styling}. Valid options are {self.style_options}")
@@ -85,11 +82,10 @@ class ExcelExporter(BaseExporter[Workbook]):
85
82
  raise ValueError(f"Invalid dump_as: {dump_as}. Valid options are {self.dump_options}")
86
83
  self.styling = styling
87
84
  self._styling_level = self.style_options.index(styling)
88
- self.output_role = output_role
89
85
  self.new_model_id = new_model_id
90
86
  self.dump_as = dump_as
91
87
 
92
- def export_to_file(self, rules: Rules, filepath: Path) -> None:
88
+ def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
93
89
  """Exports transformation rules to excel file."""
94
90
  data = self.export(rules)
95
91
  try:
@@ -98,8 +94,7 @@ class ExcelExporter(BaseExporter[Workbook]):
98
94
  data.close()
99
95
  return None
100
96
 
101
- def export(self, rules: Rules) -> Workbook:
102
- rules = self._convert_to_output_role(rules, self.output_role)
97
+ def export(self, rules: VerifiedRules) -> Workbook:
103
98
  workbook = Workbook()
104
99
  # Remove default sheet named "Sheet"
105
100
  workbook.remove(workbook["Sheet"])
@@ -147,7 +142,7 @@ class ExcelExporter(BaseExporter[Workbook]):
147
142
  self,
148
143
  workbook: Workbook,
149
144
  dumped_rules: dict[str, Any],
150
- rules: Rules,
145
+ rules: VerifiedRules,
151
146
  sheet_prefix: str = "",
152
147
  ):
153
148
  for sheet_name, headers in rules.headers_by_sheet(by_alias=True).items():
@@ -279,9 +274,7 @@ class _MetadataCreator:
279
274
 
280
275
  new_metadata = self._create_new_info(now)
281
276
  if isinstance(metadata, DMSMetadata):
282
- from cognite.neat.rules.models.information._converter import (
283
- _InformationRulesConverter,
284
- )
277
+ from cognite.neat.rules.transformers._converters import _InformationRulesConverter
285
278
 
286
279
  output_metadata: DMSMetadata | InformationMetadata = _InformationRulesConverter._convert_metadata_to_dms(
287
280
  new_metadata
@@ -9,20 +9,12 @@ from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, BNode, Graph, Literal, Namespac
9
9
  from rdflib.collection import Collection as GraphCollection
10
10
 
11
11
  from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
12
- from cognite.neat.rules.analysis import InformationAnalysis
13
- from cognite.neat.rules.issues.ontology import (
14
- MetadataSheetNamespaceNotDefinedError,
15
- MissingDataModelPrefixOrNamespaceWarning,
16
- OntologyMultiDefinitionPropertyWarning,
17
- OntologyMultiDomainPropertyWarning,
18
- OntologyMultiLabeledPropertyWarning,
19
- OntologyMultiRangePropertyWarning,
20
- OntologyMultiTypePropertyWarning,
21
- PrefixMissingError,
22
- PropertiesDefinedMultipleTimesError,
23
- PropertyDefinitionsNotForSamePropertyError,
12
+ from cognite.neat.issues import MultiValueError
13
+ from cognite.neat.issues.errors import (
14
+ PropertyDefinitionDuplicatedError,
24
15
  )
25
- from cognite.neat.rules.models import DMSRules
16
+ from cognite.neat.issues.warnings import PropertyDefinitionDuplicatedWarning
17
+ from cognite.neat.rules.analysis import InformationAnalysis
26
18
  from cognite.neat.rules.models.data_types import DataType
27
19
  from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
28
20
  from cognite.neat.rules.models.information import (
@@ -31,43 +23,40 @@ from cognite.neat.rules.models.information import (
31
23
  InformationProperty,
32
24
  InformationRules,
33
25
  )
34
- from cognite.neat.utils.auxiliary import generate_exception_report
35
26
  from cognite.neat.utils.rdf_ import remove_namespace_from_uri
36
27
 
37
28
  from ._base import BaseExporter
38
- from ._validation import are_properties_redefined
29
+ from ._validation import duplicated_properties
39
30
 
40
31
  if sys.version_info >= (3, 11):
41
32
  from typing import Self
42
33
  else:
43
34
  from typing_extensions import Self
44
35
 
45
- from cognite.neat.rules._shared import Rules
46
36
 
47
-
48
- class GraphExporter(BaseExporter[Graph], ABC):
49
- def export_to_file(self, rules: Rules, filepath: Path) -> None:
37
+ class GraphExporter(BaseExporter[InformationRules, Graph], ABC):
38
+ def export_to_file(self, rules: InformationRules, filepath: Path) -> None:
50
39
  self.export(rules).serialize(destination=filepath, encoding=self._encoding, newline=self._new_line)
51
40
 
52
41
 
53
42
  class OWLExporter(GraphExporter):
54
- """Exports rules to an OWL ontology."""
43
+ """Exports verified information rules to an OWL ontology."""
55
44
 
56
- def export(self, rules: Rules) -> Graph:
45
+ def export(self, rules: InformationRules) -> Graph:
57
46
  return Ontology.from_rules(rules).as_owl()
58
47
 
59
48
 
60
49
  class SHACLExporter(GraphExporter):
61
50
  """Exports rules to a SHACL graph."""
62
51
 
63
- def export(self, rules: Rules) -> Graph:
52
+ def export(self, rules: InformationRules) -> Graph:
64
53
  return Ontology.from_rules(rules).as_shacl()
65
54
 
66
55
 
67
56
  class SemanticDataModelExporter(GraphExporter):
68
- """Exports rules to a semantic data model."""
57
+ """Exports verified information rules to a semantic data model."""
69
58
 
70
- def export(self, rules: Rules) -> Graph:
59
+ def export(self, rules: InformationRules) -> Graph:
71
60
  return Ontology.from_rules(rules).as_semantic_data_model()
72
61
 
73
62
 
@@ -94,34 +83,30 @@ class Ontology(OntologyModel):
94
83
  prefixes: dict[str, Namespace]
95
84
 
96
85
  @classmethod
97
- def from_rules(cls, input_rules: Rules) -> Self:
86
+ def from_rules(cls, rules: InformationRules) -> Self:
98
87
  """
99
88
  Generates an ontology from a set of transformation rules.
100
89
 
101
90
  Args:
102
- input_rules: The rules to generate the ontology from.
91
+ rules: The rules to generate the ontology from.
103
92
 
104
93
  Returns:
105
94
  An instance of Ontology.
106
95
  """
107
- if isinstance(input_rules, InformationRules):
108
- rules = input_rules
109
- elif isinstance(input_rules, DMSRules):
110
- rules = input_rules.as_information_rules()
111
- else:
112
- raise ValueError(f"{type(input_rules).__name__} cannot be exported to Ontology")
113
-
114
- properties_redefined, redefinition_warnings = are_properties_redefined(rules, return_report=True)
115
- if properties_redefined:
116
- raise PropertiesDefinedMultipleTimesError(
117
- report=generate_exception_report(redefinition_warnings)
118
- ).as_exception()
119
-
120
- if rules.prefixes is None:
121
- raise PrefixMissingError().as_exception()
122
-
123
- if rules.metadata.namespace is None:
124
- raise MissingDataModelPrefixOrNamespaceWarning()
96
+ if duplicates := duplicated_properties(rules.properties):
97
+ errors = []
98
+ for (class_, property_), definitions in duplicates.items():
99
+ errors.append(
100
+ PropertyDefinitionDuplicatedError(
101
+ class_,
102
+ "class",
103
+ property_,
104
+ frozenset({str(definition[1].value_type) for definition in definitions}),
105
+ tuple(definition[0] for definition in definitions),
106
+ "rows",
107
+ )
108
+ )
109
+ raise MultiValueError(errors)
125
110
 
126
111
  class_dict = InformationAnalysis(rules).as_class_dict()
127
112
  return cls(
@@ -184,9 +169,6 @@ class Ontology(OntologyModel):
184
169
  for prefix, namespace in self.prefixes.items():
185
170
  owl.bind(prefix, namespace)
186
171
 
187
- if self.metadata.namespace is None:
188
- raise MetadataSheetNamespaceNotDefinedError().as_exception()
189
-
190
172
  owl.add((URIRef(self.metadata.namespace), RDF.type, OWL.Ontology))
191
173
  for property_ in self.properties:
192
174
  for triple in property_.triples:
@@ -233,8 +215,6 @@ class OWLMetadata(InformationMetadata):
233
215
  @property
234
216
  def triples(self) -> list[tuple]:
235
217
  # Mandatory triples originating from Metadata mandatory fields
236
- if self.namespace is None:
237
- raise MetadataSheetNamespaceNotDefinedError().as_exception()
238
218
  triples: list[tuple] = [
239
219
  (URIRef(self.namespace), DCTERMS.hasVersion, Literal(self.version)),
240
220
  (URIRef(self.namespace), OWL.versionInfo, Literal(self.version)),
@@ -323,16 +303,17 @@ class OWLProperty(OntologyModel):
323
303
  range_: set[URIRef]
324
304
  namespace: Namespace
325
305
 
326
- @staticmethod
327
- def same_property_id(definitions: list[InformationProperty]) -> bool:
328
- return len({definition.property_ for definition in definitions}) == 1
329
-
330
306
  @classmethod
331
307
  def from_list_of_properties(cls, definitions: list[InformationProperty], namespace: Namespace) -> "OWLProperty":
332
308
  """Here list of properties is a list of properties with the same id, but different definitions."""
333
-
334
- if not cls.same_property_id(definitions):
335
- raise PropertyDefinitionsNotForSamePropertyError().as_exception()
309
+ property_ids = {definition.property_ for definition in definitions}
310
+ if len(property_ids) != 1:
311
+ raise PropertyDefinitionDuplicatedError(
312
+ definitions[0].class_,
313
+ "class",
314
+ definitions[0].property_,
315
+ frozenset(property_ids),
316
+ )
336
317
 
337
318
  owl_property = cls.model_construct(
338
319
  id_=namespace[definitions[0].property_],
@@ -363,8 +344,15 @@ class OWLProperty(OntologyModel):
363
344
  def is_multi_type(cls, v, info: ValidationInfo):
364
345
  if len(v) > 1:
365
346
  warnings.warn(
366
- OntologyMultiTypePropertyWarning(
367
- remove_namespace_from_uri(info.data["id_"]), [remove_namespace_from_uri(t) for t in v]
347
+ PropertyDefinitionDuplicatedWarning(
348
+ remove_namespace_from_uri(info.data["id"]),
349
+ "class",
350
+ "type",
351
+ frozenset({remove_namespace_from_uri(t) for t in v}),
352
+ "This warning occurs when a same property is define for two object/classes where"
353
+ " its expected value type is different in one definition, e.g. acts as an edge, while in "
354
+ "other definition acts as and attribute",
355
+ "If a property takes different value types for different objects, simply define new property",
368
356
  ),
369
357
  stacklevel=2,
370
358
  )
@@ -374,8 +362,14 @@ class OWLProperty(OntologyModel):
374
362
  def is_multi_range(cls, v, info: ValidationInfo):
375
363
  if len(v) > 1:
376
364
  warnings.warn(
377
- OntologyMultiRangePropertyWarning(
378
- remove_namespace_from_uri(info.data["id_"]), [remove_namespace_from_uri(t) for t in v]
365
+ PropertyDefinitionDuplicatedWarning(
366
+ remove_namespace_from_uri(info.data["id_"]),
367
+ "class",
368
+ "range",
369
+ frozenset({remove_namespace_from_uri(t) for t in v}),
370
+ "This warning occurs when a property takes range of "
371
+ "values which consists of union of multiple value types.",
372
+ "If value types for different objects, simply define new property",
379
373
  ),
380
374
  stacklevel=2,
381
375
  )
@@ -385,8 +379,15 @@ class OWLProperty(OntologyModel):
385
379
  def is_multi_domain(cls, v, info: ValidationInfo):
386
380
  if len(v) > 1:
387
381
  warnings.warn(
388
- OntologyMultiDomainPropertyWarning(
389
- remove_namespace_from_uri(info.data["id_"]), [remove_namespace_from_uri(t) for t in v]
382
+ PropertyDefinitionDuplicatedWarning(
383
+ remove_namespace_from_uri(info.data["id_"]),
384
+ "class",
385
+ "domain",
386
+ frozenset({remove_namespace_from_uri(t) for t in v}),
387
+ "This warning occurs when a same property is define for two object/classes where"
388
+ " its expected value type is different in one definition, e.g. acts as an edge, while in "
389
+ "other definition acts as and attribute",
390
+ "If value types for different objects, simply define new property",
390
391
  ),
391
392
  stacklevel=2,
392
393
  )
@@ -396,7 +397,13 @@ class OWLProperty(OntologyModel):
396
397
  def has_multi_name(cls, v, info: ValidationInfo):
397
398
  if len(v) > 1:
398
399
  warnings.warn(
399
- OntologyMultiLabeledPropertyWarning(remove_namespace_from_uri(info.data["id_"]), v),
400
+ PropertyDefinitionDuplicatedWarning(
401
+ remove_namespace_from_uri(info.data["id_"]),
402
+ "class",
403
+ "label",
404
+ frozenset(v),
405
+ f"Only the first label (name) will be used, {v[0]}",
406
+ ),
400
407
  stacklevel=2,
401
408
  )
402
409
  return v
@@ -405,7 +412,13 @@ class OWLProperty(OntologyModel):
405
412
  def has_multi_comment(cls, v, info: ValidationInfo):
406
413
  if len(v) > 1:
407
414
  warnings.warn(
408
- OntologyMultiDefinitionPropertyWarning(remove_namespace_from_uri(info.data["id_"])),
415
+ PropertyDefinitionDuplicatedWarning(
416
+ remove_namespace_from_uri(info.data["id_"]),
417
+ "class",
418
+ "comment",
419
+ frozenset(v),
420
+ "All definitions will be concatenated to form a single definition.",
421
+ ),
409
422
  stacklevel=2,
410
423
  )
411
424
  return v
@@ -5,14 +5,13 @@ from typing import Literal, get_args
5
5
 
6
6
  import yaml
7
7
 
8
- from cognite.neat.rules._shared import Rules
9
- from cognite.neat.rules.models import RoleTypes
8
+ from cognite.neat.rules._shared import VerifiedRules
10
9
 
11
10
  from ._base import BaseExporter
12
11
 
13
12
 
14
- class YAMLExporter(BaseExporter[str]):
15
- """Export rules to YAML.
13
+ class YAMLExporter(BaseExporter[VerifiedRules, str]):
14
+ """Export rules (Information, DMS or Domain) to YAML.
16
15
 
17
16
  Args:
18
17
  files: The number of files to output. Defaults to "single".
@@ -38,16 +37,15 @@ class YAMLExporter(BaseExporter[str]):
38
37
  file_option = get_args(Files)
39
38
  format_option = get_args(Format)
40
39
 
41
- def __init__(self, files: Files = "single", output: Format = "yaml", output_role: RoleTypes | None = None):
40
+ def __init__(self, files: Files = "single", output: Format = "yaml"):
42
41
  if files not in self.file_option:
43
42
  raise ValueError(f"Invalid files: {files}. Valid options are {self.file_option}")
44
43
  if output not in self.format_option:
45
44
  raise ValueError(f"Invalid output: {output}. Valid options are {self.format_option}")
46
45
  self.files = files
47
46
  self.output = output
48
- self.output_role = output_role
49
47
 
50
- def export_to_file(self, rules: Rules, filepath: Path) -> None:
48
+ def export_to_file(self, rules: VerifiedRules, filepath: Path) -> None:
51
49
  """Exports transformation rules to YAML/JSON file(s)."""
52
50
  if self.files == "single":
53
51
  if filepath.suffix != f".{self.output}":
@@ -57,7 +55,7 @@ class YAMLExporter(BaseExporter[str]):
57
55
  else:
58
56
  raise NotImplementedError(f"Exporting to {self.files} files is not supported")
59
57
 
60
- def export(self, rules: Rules) -> str:
58
+ def export(self, rules: VerifiedRules) -> str:
61
59
  """Export rules to YAML (or JSON) format.
62
60
 
63
61
  Args:
@@ -66,7 +64,6 @@ class YAMLExporter(BaseExporter[str]):
66
64
  Returns:
67
65
  str: The rules in YAML (or JSON) format.
68
66
  """
69
- rules = self._convert_to_output_role(rules, self.output_role)
70
67
  # model_dump_json ensures that the output is in JSON format,
71
68
  # if we don't do this, we will get Enums and other types that are not serializable to YAML
72
69
  json_output = rules.dump(mode="json")