cognite-neat 0.103.0__py3-none-any.whl → 0.104.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (64) hide show
  1. cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
  2. cognite/neat/_graph/transformers/_base.py +109 -1
  3. cognite/neat/_graph/transformers/_classic_cdf.py +4 -0
  4. cognite/neat/_graph/transformers/_prune_graph.py +103 -47
  5. cognite/neat/_graph/transformers/_rdfpath.py +41 -17
  6. cognite/neat/_graph/transformers/_value_type.py +119 -154
  7. cognite/neat/_issues/_base.py +35 -8
  8. cognite/neat/_issues/warnings/_resources.py +1 -1
  9. cognite/neat/_rules/_shared.py +18 -34
  10. cognite/neat/_rules/exporters/_base.py +28 -2
  11. cognite/neat/_rules/exporters/_rules2dms.py +4 -0
  12. cognite/neat/_rules/exporters/_rules2excel.py +11 -0
  13. cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
  14. cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
  15. cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
  16. cognite/neat/_rules/importers/_base.py +9 -0
  17. cognite/neat/_rules/importers/_dms2rules.py +17 -5
  18. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
  19. cognite/neat/_rules/importers/_rdf/_base.py +10 -8
  20. cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
  21. cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
  22. cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
  23. cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
  24. cognite/neat/_rules/importers/_yaml2rules.py +21 -7
  25. cognite/neat/_rules/models/_base_input.py +1 -1
  26. cognite/neat/_rules/models/_base_rules.py +5 -0
  27. cognite/neat/_rules/models/dms/_rules.py +4 -0
  28. cognite/neat/_rules/models/dms/_rules_input.py +9 -0
  29. cognite/neat/_rules/models/dms/_validation.py +2 -0
  30. cognite/neat/_rules/models/entities/_single_value.py +25 -5
  31. cognite/neat/_rules/models/information/_rules.py +4 -0
  32. cognite/neat/_rules/models/information/_rules_input.py +9 -0
  33. cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
  34. cognite/neat/_rules/transformers/__init__.py +5 -4
  35. cognite/neat/_rules/transformers/_base.py +41 -65
  36. cognite/neat/_rules/transformers/_converters.py +149 -62
  37. cognite/neat/_rules/transformers/_mapping.py +17 -12
  38. cognite/neat/_rules/transformers/_verification.py +50 -37
  39. cognite/neat/_session/_base.py +32 -121
  40. cognite/neat/_session/_inspect.py +3 -3
  41. cognite/neat/_session/_mapping.py +17 -105
  42. cognite/neat/_session/_prepare.py +36 -254
  43. cognite/neat/_session/_read.py +11 -130
  44. cognite/neat/_session/_set.py +6 -30
  45. cognite/neat/_session/_show.py +49 -30
  46. cognite/neat/_session/_state.py +49 -107
  47. cognite/neat/_session/_to.py +42 -31
  48. cognite/neat/_shared.py +23 -2
  49. cognite/neat/_store/_provenance.py +3 -82
  50. cognite/neat/_store/_rules_store.py +372 -10
  51. cognite/neat/_store/exceptions.py +23 -0
  52. cognite/neat/_utils/graph_transformations_report.py +36 -0
  53. cognite/neat/_utils/io_.py +11 -0
  54. cognite/neat/_utils/rdf_.py +8 -0
  55. cognite/neat/_utils/spreadsheet.py +5 -4
  56. cognite/neat/_version.py +1 -1
  57. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +7 -7
  58. cognite/neat/_workflows/steps/lib/current/rules_importer.py +24 -99
  59. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/METADATA +1 -1
  60. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/RECORD +63 -61
  61. cognite/neat/_rules/transformers/_pipelines.py +0 -70
  62. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/LICENSE +0 -0
  63. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/WHEEL +0 -0
  64. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/entry_points.txt +0 -0
@@ -19,7 +19,7 @@ from cognite.client.data_classes.data_modeling.views import (
19
19
  from cognite.client.utils import ms_to_datetime
20
20
 
21
21
  from cognite.neat._client import NeatClient
22
- from cognite.neat._issues import IssueList, NeatIssue
22
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
23
23
  from cognite.neat._issues.errors import FileTypeUnexpectedError, ResourceMissingIdentifierError, ResourceRetrievalError
24
24
  from cognite.neat._issues.warnings import (
25
25
  PropertyNotFoundWarning,
@@ -89,6 +89,14 @@ class DMSImporter(BaseImporter[DMSInputRules]):
89
89
  continue
90
90
  self._all_containers_by_id[container.as_id()] = container
91
91
 
92
+ @property
93
+ def description(self) -> str:
94
+ if self.root_schema.data_model is not None:
95
+ identifier = f"{self.root_schema.data_model.as_id().as_tuple()!s}"
96
+ else:
97
+ identifier = "Unknown"
98
+ return f"DMS Data model {identifier} read as unverified data model"
99
+
92
100
  @classmethod
93
101
  def from_data_model_id(
94
102
  cls,
@@ -197,12 +205,14 @@ class DMSImporter(BaseImporter[DMSInputRules]):
197
205
  if self.issue_list.has_errors:
198
206
  # In case there were errors during the import, the to_rules method will return None
199
207
  self._end = datetime.now(timezone.utc)
200
- return ReadRules(None, self.issue_list, {})
208
+ self.issue_list.trigger_warnings()
209
+ raise MultiValueError(self.issue_list.errors)
201
210
 
202
211
  if not self.root_schema.data_model:
203
212
  self.issue_list.append(ResourceMissingIdentifierError("data model", type(self.root_schema).__name__))
204
213
  self._end = datetime.now(timezone.utc)
205
- return ReadRules(None, self.issue_list, {})
214
+ self.issue_list.trigger_warnings()
215
+ raise MultiValueError(self.issue_list.errors)
206
216
 
207
217
  model = self.root_schema.data_model
208
218
 
@@ -213,8 +223,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
213
223
  )
214
224
 
215
225
  self._end = datetime.now(timezone.utc)
216
-
217
- return ReadRules(user_rules, self.issue_list, {})
226
+ self.issue_list.trigger_warnings()
227
+ if self.issue_list.has_errors:
228
+ raise MultiValueError(self.issue_list.errors)
229
+ return ReadRules(user_rules, {})
218
230
 
219
231
  def _create_rule_components(
220
232
  self,
@@ -5,7 +5,7 @@ from pathlib import Path
5
5
 
6
6
  from pydantic import ValidationError
7
7
 
8
- from cognite.neat._issues import IssueList, NeatIssue
8
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
9
9
  from cognite.neat._issues.warnings import (
10
10
  FileItemNotSupportedWarning,
11
11
  FileMissingRequiredFieldWarning,
@@ -144,5 +144,8 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
144
144
  properties=converter.properties,
145
145
  classes=converter.classes,
146
146
  )
147
+ converter.issues.trigger_warnings()
148
+ if converter.issues.has_errors:
149
+ raise MultiValueError(converter.issues.errors)
147
150
 
148
- return ReadRules(rules, converter.issues, {})
151
+ return ReadRules(rules, {})
@@ -5,7 +5,7 @@ from cognite.client import data_modeling as dm
5
5
  from rdflib import Graph, Namespace, URIRef
6
6
 
7
7
  from cognite.neat._constants import get_default_prefixes_and_namespaces
8
- from cognite.neat._issues import IssueList
8
+ from cognite.neat._issues import IssueList, MultiValueError
9
9
  from cognite.neat._issues.errors import FileReadError
10
10
  from cognite.neat._issues.errors._general import NeatValueError
11
11
  from cognite.neat._rules._shared import ReadRules
@@ -13,9 +13,7 @@ from cognite.neat._rules.importers._base import BaseImporter
13
13
  from cognite.neat._rules.models._base_rules import RoleTypes
14
14
  from cognite.neat._rules.models.data_types import AnyURI
15
15
  from cognite.neat._rules.models.entities import UnknownEntity
16
- from cognite.neat._rules.models.information import (
17
- InformationInputRules,
18
- )
16
+ from cognite.neat._rules.models.information import InformationInputRules
19
17
  from cognite.neat._store import NeatGraphStore
20
18
  from cognite.neat._utils.rdf_ import get_namespace
21
19
 
@@ -50,6 +48,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
50
48
  max_number_of_instance: int,
51
49
  non_existing_node_type: UnknownEntity | AnyURI,
52
50
  language: str,
51
+ source_name: str = "Unknown",
53
52
  ) -> None:
54
53
  self.issue_list = issue_list
55
54
  self.graph = graph
@@ -60,6 +59,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
60
59
  self.max_number_of_instance = max_number_of_instance
61
60
  self.non_existing_node_type = non_existing_node_type
62
61
  self.language = language
62
+ self.source_name = source_name
63
63
 
64
64
  @classmethod
65
65
  def from_graph_store(
@@ -87,6 +87,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
87
87
  max_number_of_instance: int = -1,
88
88
  non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
89
89
  language: str = "en",
90
+ source_name: str = "Unknown",
90
91
  ):
91
92
  issue_list = IssueList(title=f"{cls.__name__} issues")
92
93
 
@@ -107,6 +108,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
107
108
  max_number_of_instance=max_number_of_instance,
108
109
  non_existing_node_type=non_existing_node_type,
109
110
  language=language,
111
+ source_name=source_name,
110
112
  )
111
113
 
112
114
  def to_rules(
@@ -115,16 +117,16 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
115
117
  """
116
118
  Creates `Rules` object from the data for target role.
117
119
  """
118
-
119
120
  if self.issue_list.has_errors:
120
121
  # In case there were errors during the import, the to_rules method will return None
121
- return ReadRules(None, self.issue_list, {})
122
+ self.issue_list.trigger_warnings()
123
+ raise MultiValueError(self.issue_list.errors)
122
124
 
123
125
  rules_dict = self._to_rules_components()
124
126
 
125
127
  rules = InformationInputRules.load(rules_dict)
126
-
127
- return ReadRules(rules, self.issue_list, {})
128
+ self.issue_list.trigger_warnings()
129
+ return ReadRules(rules, {})
128
130
 
129
131
  def _to_rules_components(self) -> dict:
130
132
  raise NotImplementedError()
@@ -74,6 +74,10 @@ PROPERTIES_QUERY = """
74
74
  class IMFImporter(BaseRDFImporter):
75
75
  """Convert IMF Types provided as SHACL shapes to Input Rules."""
76
76
 
77
+ @property
78
+ def description(self) -> str:
79
+ return f"IMF Types {self.source_name} read as unverified data model"
80
+
77
81
  def _to_rules_components(
78
82
  self,
79
83
  ) -> dict:
@@ -16,6 +16,7 @@ from cognite.neat._rules.models.information import (
16
16
  InformationMetadata,
17
17
  )
18
18
  from cognite.neat._store import NeatGraphStore
19
+ from cognite.neat._store._provenance import INSTANCES_ENTITY
19
20
  from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
20
21
 
21
22
  from ._base import DEFAULT_NON_EXISTING_NODE_TYPE, BaseRDFImporter
@@ -94,6 +95,7 @@ class InferenceImporter(BaseRDFImporter):
94
95
  max_number_of_instance: int = -1,
95
96
  non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
96
97
  language: str = "en",
98
+ source_name: str = "Unknown",
97
99
  ) -> "InferenceImporter":
98
100
  return super().from_file(
99
101
  filepath,
@@ -101,6 +103,7 @@ class InferenceImporter(BaseRDFImporter):
101
103
  max_number_of_instance,
102
104
  non_existing_node_type,
103
105
  language,
106
+ source_name=source_name,
104
107
  )
105
108
 
106
109
  @classmethod
@@ -283,3 +286,7 @@ class InferenceImporter(BaseRDFImporter):
283
286
  updated=now,
284
287
  description="Inferred model from knowledge graph",
285
288
  )
289
+
290
+ @property
291
+ def source_uri(self) -> URIRef:
292
+ return INSTANCES_ENTITY.id_
@@ -78,3 +78,7 @@ class OWLImporter(BaseRDFImporter):
78
78
  }
79
79
 
80
80
  return components
81
+
82
+ @property
83
+ def description(self) -> str:
84
+ return f"Ontology {self.source_name} read as unverified data model"
@@ -11,9 +11,9 @@ from typing import Literal, cast
11
11
  import pandas as pd
12
12
  from cognite.client.utils._importing import local_import
13
13
  from pandas import ExcelFile
14
- from rdflib import Namespace
14
+ from rdflib import Namespace, URIRef
15
15
 
16
- from cognite.neat._issues import IssueList
16
+ from cognite.neat._issues import IssueList, MultiValueError
17
17
  from cognite.neat._issues.errors import (
18
18
  FileMissingRequiredFieldError,
19
19
  FileNotFoundNeatError,
@@ -253,18 +253,19 @@ class ExcelImporter(BaseImporter[T_InputRules]):
253
253
  def to_rules(self) -> ReadRules[T_InputRules]:
254
254
  issue_list = IssueList(title=f"'{self.filepath.name}'")
255
255
  if not self.filepath.exists():
256
- issue_list.append(FileNotFoundNeatError(self.filepath))
257
- return ReadRules(None, issue_list, {})
256
+ raise FileNotFoundNeatError(self.filepath)
258
257
 
259
258
  with pd.ExcelFile(self.filepath) as excel_file:
260
259
  user_reader = SpreadsheetReader(issue_list)
261
260
 
262
261
  user_read = user_reader.read(excel_file, self.filepath)
263
- if user_read is None or issue_list.has_errors:
264
- return ReadRules(None, issue_list, {})
265
262
 
263
+ issue_list.trigger_warnings()
266
264
  if issue_list.has_errors:
267
- return ReadRules(None, issue_list, {})
265
+ raise MultiValueError(issue_list.errors)
266
+
267
+ if user_read is None:
268
+ return ReadRules(None, {})
268
269
 
269
270
  sheets = user_read.sheets
270
271
  original_role = user_read.role
@@ -272,7 +273,15 @@ class ExcelImporter(BaseImporter[T_InputRules]):
272
273
 
273
274
  rules_cls = INPUT_RULES_BY_ROLE[original_role]
274
275
  rules = cast(T_InputRules, rules_cls.load(sheets))
275
- return ReadRules(rules, issue_list, {"read_info_by_sheet": read_info_by_sheet})
276
+ return ReadRules(rules, {"read_info_by_sheet": read_info_by_sheet})
277
+
278
+ @property
279
+ def description(self) -> str:
280
+ return f"Excel file {self.filepath.name} read as unverified data model"
281
+
282
+ @property
283
+ def source_uri(self) -> URIRef:
284
+ return URIRef(f"file://{self.filepath.name}")
276
285
 
277
286
 
278
287
  class GoogleSheetImporter(BaseImporter[T_InputRules]):
@@ -3,7 +3,7 @@ from typing import Any, cast
3
3
 
4
4
  import yaml
5
5
 
6
- from cognite.neat._issues import IssueList, NeatIssue
6
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
7
7
  from cognite.neat._issues.errors import (
8
8
  FileMissingRequiredFieldError,
9
9
  FileNotAFileError,
@@ -36,24 +36,32 @@ class YAMLImporter(BaseImporter[T_InputRules]):
36
36
  raw_data: dict[str, Any],
37
37
  read_issues: list[NeatIssue] | None = None,
38
38
  filepaths: list[Path] | None = None,
39
+ source_name: str = "Unknown",
39
40
  ) -> None:
40
41
  self.raw_data = raw_data
41
42
  self._read_issues = IssueList(read_issues)
42
43
  self._filepaths = filepaths
44
+ self._source_name = source_name
45
+
46
+ @property
47
+ def description(self) -> str:
48
+ return f"YAML file {self._source_name} read as unverified data model"
43
49
 
44
50
  @classmethod
45
- def from_file(cls, filepath: Path):
51
+ def from_file(cls, filepath: Path, source_name: str = "Unknown") -> "YAMLImporter":
46
52
  if not filepath.exists():
47
53
  return cls({}, [FileNotFoundNeatError(filepath)])
48
54
  elif not filepath.is_file():
49
55
  return cls({}, [FileNotAFileError(filepath)])
50
56
  elif filepath.suffix not in [".yaml", ".yml"]:
51
57
  return cls({}, [FileTypeUnexpectedError(filepath, frozenset([".yaml", ".yml"]))])
52
- return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
58
+ return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath], source_name=source_name)
53
59
 
54
60
  def to_rules(self) -> ReadRules[T_InputRules]:
55
61
  if self._read_issues.has_errors or not self.raw_data:
56
- return ReadRules(None, self._read_issues, {})
62
+ self._read_issues.trigger_warnings()
63
+ raise MultiValueError(self._read_issues.errors)
64
+
57
65
  issue_list = IssueList(title="YAML Importer", issues=self._read_issues)
58
66
 
59
67
  if not self._filepaths:
@@ -69,13 +77,15 @@ class YAMLImporter(BaseImporter[T_InputRules]):
69
77
 
70
78
  if "metadata" not in self.raw_data:
71
79
  self._read_issues.append(FileMissingRequiredFieldError(metadata_file, "section", "metadata"))
72
- return ReadRules(None, self._read_issues, {})
80
+ issue_list.trigger_warnings()
81
+ raise MultiValueError(self._read_issues.errors)
73
82
 
74
83
  metadata = self.raw_data["metadata"]
75
84
 
76
85
  if "role" not in metadata:
77
86
  self._read_issues.append(FileMissingRequiredFieldError(metadata, "metadata", "role"))
78
- return ReadRules(None, self._read_issues, {})
87
+ issue_list.trigger_warnings()
88
+ raise MultiValueError(self._read_issues.errors)
79
89
 
80
90
  role_input = RoleTypes(metadata["role"])
81
91
  role_enum = RoleTypes(role_input)
@@ -83,4 +93,8 @@ class YAMLImporter(BaseImporter[T_InputRules]):
83
93
 
84
94
  rules = cast(T_InputRules, rules_cls.load(self.raw_data))
85
95
 
86
- return ReadRules(rules, issue_list, {})
96
+ issue_list.trigger_warnings()
97
+ if self._read_issues.has_errors:
98
+ raise MultiValueError(self._read_issues.errors)
99
+
100
+ return ReadRules[T_InputRules](rules, {})
@@ -110,7 +110,7 @@ class InputRules(Generic[T_BaseRules], ABC):
110
110
  def _dataclass_fields(self) -> list[Field]:
111
111
  return list(fields(self))
112
112
 
113
- def as_rules(self) -> T_BaseRules:
113
+ def as_verified_rules(self) -> T_BaseRules:
114
114
  cls_ = self._get_verified_cls()
115
115
  return cls_.model_validate(self.dump())
116
116
 
@@ -50,6 +50,7 @@ from cognite.neat._rules.models._types import (
50
50
  )
51
51
  from cognite.neat._rules.models.data_types import DataType
52
52
  from cognite.neat._rules.models.entities import EdgeEntity, ReverseConnectionEntity, ViewEntity
53
+ from cognite.neat._utils.rdf_ import uri_display_name
53
54
 
54
55
  if sys.version_info >= (3, 11):
55
56
  from enum import StrEnum
@@ -285,6 +286,10 @@ class BaseRules(SchemaModel, ABC):
285
286
  ]
286
287
  return headers_by_sheet
287
288
 
289
+ @property
290
+ def display_name(self):
291
+ return uri_display_name(self.metadata.identifier)
292
+
288
293
  def dump(
289
294
  self,
290
295
  entities_exclude_defaults: bool = True,
@@ -447,6 +447,10 @@ class DMSRules(BaseRules):
447
447
 
448
448
  return _DMSExporter(self, instance_space, remove_cdf_spaces=remove_cdf_spaces).to_schema()
449
449
 
450
+ @classmethod
451
+ def display_type_name(cls) -> str:
452
+ return "VerifiedDMSModel"
453
+
450
454
  def _repr_html_(self) -> str:
451
455
  summary = {
452
456
  "aspect": self.metadata.aspect,
@@ -21,6 +21,7 @@ from cognite.neat._rules.models.entities import (
21
21
  load_connection,
22
22
  load_dms_value_type,
23
23
  )
24
+ from cognite.neat._utils.rdf_ import uri_display_name
24
25
 
25
26
  from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
26
27
 
@@ -289,6 +290,14 @@ class DMSInputRules(InputRules[DMSRules]):
289
290
  "Nodes": [node_type.dump(default_space) for node_type in self.nodes or []] or None,
290
291
  }
291
292
 
293
+ @classmethod
294
+ def display_type_name(cls) -> str:
295
+ return "UnverifiedDMSModel"
296
+
297
+ @property
298
+ def display_name(self):
299
+ return uri_display_name(self.metadata.identifier)
300
+
292
301
  def _repr_html_(self) -> str:
293
302
  summary = {
294
303
  "type": "Physical Data Model",
@@ -194,6 +194,8 @@ class DMSValidation:
194
194
  ) -> dict[ViewId, set[ViewId]]:
195
195
  @lru_cache
196
196
  def get_parents(child_view_id: ViewId) -> set[ViewId]:
197
+ if child_view_id not in all_views_by_id:
198
+ return set()
197
199
  child_view = all_views_by_id[child_view_id]
198
200
  parents = set(child_view.implements or [])
199
201
  for parent_id in child_view.implements or []:
@@ -201,16 +201,36 @@ class Entity(BaseModel, extra="ignore"):
201
201
  for field_name, field in self.model_fields.items()
202
202
  if (v := getattr(self, field_name)) is not None and field_name not in {"prefix", "suffix"}
203
203
  }
204
+ # We only remove the default values if all the fields are default
205
+ # For example, if we dump `cdf_cdm:CogniteAsset(version=v1)` and the default is `version=v1`,
206
+ # we should not remove it unless the space is `cdf_cdm`
207
+ to_delete: list[str] = []
208
+ is_removing_defaults = True
204
209
  if isinstance(defaults, dict):
205
210
  for key, value in defaults.items():
206
- if key in model_dump and model_dump[key] == value:
207
- del model_dump[key]
208
- # Sorting to ensure deterministic order
209
- args = ",".join(f"{k}={v}" for k, v in sorted(model_dump.items(), key=lambda x: x[0]))
210
- if self.prefix == Undefined or (isinstance(defaults, dict) and self.prefix == defaults.get("prefix")):
211
+ if key not in model_dump:
212
+ continue
213
+ if model_dump[key] == value:
214
+ to_delete.append(key)
215
+ elif isinstance(self, EdgeEntity):
216
+ # Exception is edge entity, then, we remove all the defaults we can.
217
+ continue
218
+ else:
219
+ # Not all fields are default. We should not remove any of them.
220
+ is_removing_defaults = False
221
+ break
222
+ if isinstance(defaults, dict) and self.prefix == defaults.get("prefix") and is_removing_defaults:
223
+ base_id = str(self.suffix)
224
+ elif self.prefix == Undefined:
211
225
  base_id = str(self.suffix)
212
226
  else:
227
+ is_removing_defaults = False
213
228
  base_id = f"{self.prefix}:{self.suffix!s}"
229
+ if is_removing_defaults:
230
+ for key in to_delete:
231
+ del model_dump[key]
232
+ # Sorting to ensure deterministic order
233
+ args = ",".join(f"{k}={v}" for k, v in sorted(model_dump.items(), key=lambda x: x[0]))
214
234
  if args:
215
235
  return f"{base_id}({args})"
216
236
  else:
@@ -272,6 +272,10 @@ class InformationRules(BaseRules):
272
272
 
273
273
  return _InformationRulesConverter(self).as_dms_rules()
274
274
 
275
+ @classmethod
276
+ def display_type_name(cls) -> str:
277
+ return "VerifiedInformationModel"
278
+
275
279
  def _repr_html_(self) -> str:
276
280
  summary = {
277
281
  "type": "Logical Data Model",
@@ -14,6 +14,7 @@ from cognite.neat._rules.models.entities import (
14
14
  UnknownEntity,
15
15
  load_value_type,
16
16
  )
17
+ from cognite.neat._utils.rdf_ import uri_display_name
17
18
 
18
19
  from ._rules import (
19
20
  InformationClass,
@@ -150,6 +151,14 @@ class InformationInputRules(InputRules[InformationRules]):
150
151
  Prefixes=self.prefixes,
151
152
  )
152
153
 
154
+ @classmethod
155
+ def display_type_name(cls) -> str:
156
+ return "UnverifiedInformationModel"
157
+
158
+ @property
159
+ def display_name(self):
160
+ return uri_display_name(self.metadata.identifier)
161
+
153
162
  def _repr_html_(self) -> str:
154
163
  summary = {
155
164
  "type": "Logical Data Model",
@@ -32,9 +32,6 @@ def load_classic_to_core_mapping(org_name: str, source_space: str, source_versio
32
32
  if not isinstance(read.rules, DMSInputRules):
33
33
  raise NeatValueError(f"Expected DMS rules, but got {type(read.rules).__name__}")
34
34
 
35
- verified = VerifyDMSRules(errors="raise", validate=False).transform(read)
35
+ verified = VerifyDMSRules(validate=False).transform(read)
36
36
 
37
- if verified.rules is None:
38
- raise NeatValueError("Failed to verify the rules.")
39
-
40
- return verified.rules
37
+ return verified
@@ -1,7 +1,9 @@
1
- from ._base import RulesPipeline, RulesTransformer
1
+ from ._base import RulesTransformer
2
2
  from ._converters import (
3
+ AddClassImplements,
3
4
  ConvertToRules,
4
5
  DMSToInformation,
6
+ IncludeReferenced,
5
7
  InformationToDMS,
6
8
  PrefixEntities,
7
9
  ReduceCogniteModel,
@@ -10,20 +12,19 @@ from ._converters import (
10
12
  ToExtension,
11
13
  )
12
14
  from ._mapping import AsParentPropertyId, MapOneToOne, RuleMapper
13
- from ._pipelines import ImporterPipeline
14
15
  from ._verification import VerifyAnyRules, VerifyDMSRules, VerifyInformationRules
15
16
 
16
17
  __all__ = [
18
+ "AddClassImplements",
17
19
  "AsParentPropertyId",
18
20
  "ConvertToRules",
19
21
  "DMSToInformation",
20
- "ImporterPipeline",
22
+ "IncludeReferenced",
21
23
  "InformationToDMS",
22
24
  "MapOneToOne",
23
25
  "PrefixEntities",
24
26
  "ReduceCogniteModel",
25
27
  "RuleMapper",
26
- "RulesPipeline",
27
28
  "RulesTransformer",
28
29
  "SetIDDMSModel",
29
30
  "ToCompliantEntities",
@@ -1,18 +1,12 @@
1
+ import inspect
1
2
  from abc import ABC, abstractmethod
2
- from collections.abc import MutableSequence
3
- from typing import Generic, TypeVar
3
+ from functools import lru_cache
4
+ from types import UnionType
5
+ from typing import Generic, TypeVar, Union, get_args, get_origin
4
6
 
5
7
  from cognite.neat._constants import DEFAULT_NAMESPACE
6
- from cognite.neat._issues import IssueList, NeatError
7
- from cognite.neat._issues.errors import NeatTypeError, NeatValueError
8
- from cognite.neat._rules._shared import (
9
- InputRules,
10
- JustRules,
11
- MaybeRules,
12
- OutRules,
13
- Rules,
14
- VerifiedRules,
15
- )
8
+ from cognite.neat._rules._shared import ReadRules, Rules
9
+ from cognite.neat._rules.models import DMSInputRules, InformationInputRules
16
10
  from cognite.neat._store._provenance import Agent as ProvenanceAgent
17
11
 
18
12
  T_RulesIn = TypeVar("T_RulesIn", bound=Rules)
@@ -20,69 +14,51 @@ T_RulesOut = TypeVar("T_RulesOut", bound=Rules)
20
14
 
21
15
 
22
16
  class RulesTransformer(ABC, Generic[T_RulesIn, T_RulesOut]):
23
- """This is the base class for all rule transformers.
24
-
25
- Note transformers follow the functional pattern Monad
26
- https://en.wikipedia.org/wiki/Monad_(functional_programming)
27
- """
17
+ """This is the base class for all rule transformers."""
28
18
 
29
19
  @abstractmethod
30
- def transform(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> OutRules[T_RulesOut]:
20
+ def transform(self, rules: T_RulesIn) -> T_RulesOut:
31
21
  """Transform the input rules into the output rules."""
32
22
  raise NotImplementedError()
33
23
 
34
- def try_transform(self, rules: MaybeRules[T_RulesIn]) -> MaybeRules[T_RulesOut]:
35
- """Try to transform the input rules into the output rules."""
36
- try:
37
- result = self.transform(rules)
38
- except NeatError:
39
- # Any error caught during transformation will be returned as issues
40
- return MaybeRules(None, rules.issues)
41
- issues = IssueList(rules.issues, title=rules.issues.title)
42
- if isinstance(result, MaybeRules):
43
- issues.extend(result.issues)
44
- return MaybeRules(result.get_rules(), issues)
45
-
46
- @classmethod
47
- def _to_rules(cls, rules: T_RulesIn | OutRules[T_RulesIn]) -> T_RulesIn:
48
- if isinstance(rules, JustRules):
49
- return rules.rules
50
- elif isinstance(rules, MaybeRules):
51
- if rules.rules is None:
52
- raise NeatValueError("Rules is missing cannot convert")
53
- return rules.rules
54
- elif isinstance(rules, VerifiedRules | InputRules):
55
- return rules # type: ignore[return-value]
56
- else:
57
- raise NeatTypeError(f"Unsupported type: {type(rules)}")
58
-
59
24
  @property
60
25
  def agent(self) -> ProvenanceAgent:
61
26
  """Provenance agent for the importer."""
62
27
  return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
63
28
 
29
+ @property
30
+ def description(self) -> str:
31
+ """Get the description of the transformer."""
32
+ return "MISSING DESCRIPTION"
64
33
 
65
- class RulesPipeline(list, MutableSequence[RulesTransformer], Generic[T_RulesIn, T_RulesOut]):
66
- def transform(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> OutRules[T_RulesOut]:
67
- """Transform the input rules into the output rules."""
68
- for transformer in self:
69
- rules = transformer.transform(rules)
70
- return rules # type: ignore[return-value]
34
+ def is_valid_input(self, rules: T_RulesIn) -> bool:
35
+ """Check if the input rules are valid."""
36
+ types = self.transform_type_hint()
37
+ for type_ in types:
38
+ if get_origin(type_) is ReadRules:
39
+ inner = get_args(type_)[0]
40
+ if isinstance(rules, ReadRules) and isinstance(rules.rules, inner):
41
+ return True
42
+ elif isinstance(rules, type_):
43
+ return True
44
+ return False
45
+
46
+ @classmethod
47
+ @lru_cache(maxsize=1)
48
+ def transform_type_hint(cls) -> tuple[type, ...]:
49
+ # This is an expensive operation, so we cache the result
50
+ signature = inspect.signature(cls.transform)
51
+ annotation = signature.parameters["rules"].annotation
52
+ if isinstance(annotation, TypeVar):
53
+ if annotation.__bound__ is None:
54
+ raise TypeError(f"TypeVar {annotation} must be bound to a type.")
55
+ annotation = annotation.__bound__
56
+ # The annotation can be a type or a generic
57
+ if get_origin(annotation) in [UnionType, Union]:
58
+ return get_args(annotation)
71
59
 
72
- def try_transform(self, rules: MaybeRules[T_RulesIn]) -> MaybeRules[T_RulesOut]:
73
- """Try to transform the input rules into the output rules."""
74
- for transformer in self:
75
- rules = transformer.try_transform(rules)
76
- return rules # type: ignore[return-value]
60
+ if get_origin(annotation) is ReadRules and isinstance(get_args(annotation)[0], TypeVar):
61
+ # Hardcoded for now, as we only have two types of ReadRules
62
+ return ReadRules[DMSInputRules], ReadRules[InformationInputRules]
77
63
 
78
- def run(self, rules: T_RulesIn | OutRules[T_RulesIn]) -> T_RulesOut:
79
- """Run the pipeline from the input rules to the output rules."""
80
- output = self.transform(rules)
81
- if isinstance(output, MaybeRules):
82
- if output.rules is None:
83
- raise NeatValueError(f"Rule transformation failed: {output.issues}")
84
- return output.rules
85
- elif isinstance(output, JustRules):
86
- return output.rules
87
- else:
88
- raise NeatTypeError(f"Rule transformation failed: {output}")
64
+ return (annotation,)