cognite-neat 0.121.1__py3-none-any.whl → 0.121.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (97) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/core/_client/_api/statistics.py +91 -0
  3. cognite/neat/core/_client/_api_client.py +2 -0
  4. cognite/neat/core/_client/data_classes/statistics.py +125 -0
  5. cognite/neat/core/_client/testing.py +4 -0
  6. cognite/neat/core/_constants.py +6 -7
  7. cognite/neat/core/_data_model/_constants.py +23 -16
  8. cognite/neat/core/_data_model/_shared.py +33 -17
  9. cognite/neat/core/_data_model/analysis/__init__.py +2 -2
  10. cognite/neat/core/_data_model/analysis/_base.py +186 -183
  11. cognite/neat/core/_data_model/catalog/__init__.py +1 -1
  12. cognite/neat/core/_data_model/exporters/__init__.py +5 -5
  13. cognite/neat/core/_data_model/exporters/_base.py +10 -8
  14. cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
  15. cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
  16. cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +4 -4
  17. cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
  18. cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
  19. cognite/neat/core/_data_model/importers/__init__.py +6 -6
  20. cognite/neat/core/_data_model/importers/_base.py +8 -6
  21. cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
  22. cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +40 -20
  23. cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
  24. cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
  25. cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
  26. cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
  27. cognite/neat/core/_data_model/importers/_rdf/_base.py +9 -9
  28. cognite/neat/core/_data_model/importers/_rdf/_imf2rules.py +15 -15
  29. cognite/neat/core/_data_model/importers/_rdf/_inference2rules.py +36 -36
  30. cognite/neat/core/_data_model/importers/_rdf/_owl2rules.py +12 -12
  31. cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
  32. cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +72 -12
  33. cognite/neat/core/_data_model/models/__init__.py +8 -8
  34. cognite/neat/core/_data_model/models/_base_unverified.py +1 -1
  35. cognite/neat/core/_data_model/models/_base_verified.py +3 -3
  36. cognite/neat/core/_data_model/models/_types.py +6 -6
  37. cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
  38. cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
  39. cognite/neat/core/_data_model/models/conceptual/_validation.py +87 -77
  40. cognite/neat/core/_data_model/models/conceptual/_verified.py +53 -51
  41. cognite/neat/core/_data_model/models/data_types.py +2 -2
  42. cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
  43. cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
  44. cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
  45. cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
  46. cognite/neat/core/_data_model/models/entities/_types.py +9 -3
  47. cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
  48. cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
  49. cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
  50. cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +71 -52
  51. cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
  52. cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +13 -11
  53. cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
  54. cognite/neat/core/_data_model/transformers/__init__.py +27 -23
  55. cognite/neat/core/_data_model/transformers/_base.py +26 -19
  56. cognite/neat/core/_data_model/transformers/_converters.py +703 -618
  57. cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
  58. cognite/neat/core/_data_model/transformers/_verification.py +63 -54
  59. cognite/neat/core/_instances/extractors/_base.py +1 -1
  60. cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +8 -8
  61. cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
  62. cognite/neat/core/_instances/extractors/_mock_graph_generator.py +98 -95
  63. cognite/neat/core/_instances/loaders/_base.py +2 -2
  64. cognite/neat/core/_instances/loaders/_rdf2dms.py +6 -6
  65. cognite/neat/core/_instances/transformers/_base.py +7 -4
  66. cognite/neat/core/_instances/transformers/_value_type.py +2 -6
  67. cognite/neat/core/_issues/_base.py +4 -4
  68. cognite/neat/core/_issues/errors/__init__.py +2 -2
  69. cognite/neat/core/_issues/errors/_wrapper.py +2 -2
  70. cognite/neat/core/_issues/warnings/_models.py +4 -4
  71. cognite/neat/core/_store/__init__.py +3 -3
  72. cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
  73. cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
  74. cognite/neat/core/_store/_provenance.py +2 -2
  75. cognite/neat/core/_store/exceptions.py +2 -2
  76. cognite/neat/core/_utils/rdf_.py +14 -0
  77. cognite/neat/core/_utils/text.py +1 -1
  78. cognite/neat/session/_base.py +22 -20
  79. cognite/neat/session/_drop.py +2 -2
  80. cognite/neat/session/_inspect.py +5 -5
  81. cognite/neat/session/_mapping.py +8 -6
  82. cognite/neat/session/_read.py +2 -2
  83. cognite/neat/session/_set.py +3 -3
  84. cognite/neat/session/_show.py +11 -11
  85. cognite/neat/session/_state.py +13 -13
  86. cognite/neat/session/_subset.py +12 -9
  87. cognite/neat/session/_template.py +13 -13
  88. cognite/neat/session/_to.py +17 -17
  89. {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
  90. {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/RECORD +95 -93
  91. cognite/neat/core/_data_model/exporters/_validation.py +0 -14
  92. cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
  93. /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
  94. /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
  95. /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
  96. {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
  97. {cognite_neat-0.121.1.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
@@ -33,7 +33,7 @@ from ._resources import (
33
33
  ResourceRetrievalError,
34
34
  )
35
35
  from ._wrapper import (
36
- ClassValueError,
36
+ ConceptValueError,
37
37
  ContainerValueError,
38
38
  EnumValueError,
39
39
  MetadataValueError,
@@ -47,7 +47,7 @@ __all__ = [
47
47
  "AuthorizationError",
48
48
  "CDFMissingClientError",
49
49
  "CDFMissingResourcesError",
50
- "ClassValueError",
50
+ "ConceptValueError",
51
51
  "ContainerValueError",
52
52
  "EnumValueError",
53
53
  "FileMissingRequiredFieldError",
@@ -67,8 +67,8 @@ class PropertyValueError(SpreadsheetListError):
67
67
 
68
68
 
69
69
  @dataclass(unsafe_hash=True)
70
- class ClassValueError(SpreadsheetListError):
71
- _name = "Classes"
70
+ class ConceptValueError(SpreadsheetListError):
71
+ _name = "Concepts"
72
72
 
73
73
 
74
74
  @dataclass(unsafe_hash=True)
@@ -93,12 +93,12 @@ class NotSupportedHasDataFilterLimitWarning(CDFNotSupportedWarning):
93
93
 
94
94
 
95
95
  @dataclass(unsafe_hash=True)
96
- class UndefinedClassWarning(UserModelingWarning):
97
- """Class {class_id} has no explicit properties defined neither implements other class"""
96
+ class UndefinedConceptWarning(UserModelingWarning):
97
+ """Class {concept_id} has no explicit properties defined neither implements other concepts"""
98
98
 
99
- fix = "Define properties for class or inherit properties by implementing another class."
99
+ fix = "Define properties for concept or inherit properties by implementing another concept."
100
100
 
101
- class_id: str
101
+ concept_id: str
102
102
 
103
103
 
104
104
  @dataclass(unsafe_hash=True)
@@ -1,4 +1,4 @@
1
- from ._graph_store import NeatGraphStore
2
- from ._rules_store import NeatRulesStore
1
+ from ._data_model import NeatDataModelStore
2
+ from ._instance import NeatInstanceStore
3
3
 
4
- __all__ = ["NeatGraphStore", "NeatRulesStore"]
4
+ __all__ = ["NeatDataModelStore", "NeatInstanceStore"]
@@ -12,15 +12,15 @@ from rdflib import URIRef
12
12
 
13
13
  from cognite.neat.core._client import NeatClient
14
14
  from cognite.neat.core._constants import DEFAULT_NAMESPACE
15
- from cognite.neat.core._data_model._shared import T_VerifiedRules, VerifiedRules
15
+ from cognite.neat.core._data_model._shared import T_VerifiedDataModel, VerifiedDataModel
16
16
  from cognite.neat.core._data_model.exporters import BaseExporter
17
17
  from cognite.neat.core._data_model.exporters._base import CDFExporter, T_Export
18
18
  from cognite.neat.core._data_model.importers import BaseImporter
19
- from cognite.neat.core._data_model.models import ConceptualDataModel, DMSRules
19
+ from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
20
20
  from cognite.neat.core._data_model.transformers import (
21
- DMSToInformation,
22
- VerifiedRulesTransformer,
23
- VerifyAnyRules,
21
+ PhysicalToConceptual,
22
+ VerifiedDataModelTransformer,
23
+ VerifyAnyDataModel,
24
24
  )
25
25
  from cognite.neat.core._instances.extractors import (
26
26
  DMSGraphExtractor,
@@ -42,19 +42,19 @@ from .exceptions import EmptyStore, InvalidActivityInput
42
42
 
43
43
 
44
44
  @dataclass(frozen=True)
45
- class RulesEntity(Entity):
46
- information: ConceptualDataModel
47
- dms: DMSRules | None = None
45
+ class DataModelEntity(Entity):
46
+ conceptual: ConceptualDataModel
47
+ physical: PhysicalDataModel | None = None
48
48
 
49
49
  @property
50
- def has_dms(self) -> bool:
51
- return self.dms is not None
50
+ def has_physical(self) -> bool:
51
+ return self.physical is not None
52
52
 
53
53
  @property
54
54
  def display_name(self) -> str:
55
- if self.dms is not None:
56
- return self.dms.display_name
57
- return self.information.display_name
55
+ if self.physical is not None:
56
+ return self.physical.display_name
57
+ return self.conceptual.display_name
58
58
 
59
59
 
60
60
  @dataclass(frozen=True)
@@ -62,9 +62,9 @@ class OutcomeEntity(Entity):
62
62
  result: UploadResultList | Path | str | URIRef
63
63
 
64
64
 
65
- class NeatRulesStore:
65
+ class NeatDataModelStore:
66
66
  def __init__(self) -> None:
67
- self.provenance = Provenance[RulesEntity]()
67
+ self.provenance = Provenance[DataModelEntity]()
68
68
  self.exports_by_source_entity_id: dict[rdflib.URIRef, list[Change[OutcomeEntity]]] = defaultdict(list)
69
69
  self._last_outcome: UploadResultList | None = None
70
70
  self._iteration_by_id: dict[Hashable, int] = {}
@@ -80,19 +80,19 @@ class NeatRulesStore:
80
80
  return calculated_hash[:8]
81
81
  return calculated_hash
82
82
 
83
- def _rules_import_verify_convert(
83
+ def _data_model_import_verify_convert(
84
84
  self,
85
85
  importer: BaseImporter,
86
86
  validate: bool,
87
87
  client: NeatClient | None = None,
88
- ) -> tuple[ConceptualDataModel, DMSRules | None]:
89
- """Action that imports rules, verifies them and optionally converts them."""
90
- read_rules = importer.to_rules()
91
- verified = VerifyAnyRules(validate, client).transform(read_rules) # type: ignore[arg-type]
88
+ ) -> tuple[ConceptualDataModel, PhysicalDataModel | None]:
89
+ """Action that imports data model, verifies them and optionally converts them."""
90
+ imported_data_model = importer.to_data_model()
91
+ verified = VerifyAnyDataModel(validate, client).transform(imported_data_model) # type: ignore[arg-type]
92
92
  if isinstance(verified, ConceptualDataModel):
93
93
  return verified, None
94
- elif isinstance(verified, DMSRules):
95
- return DMSToInformation().transform(verified), verified
94
+ elif isinstance(verified, PhysicalDataModel):
95
+ return PhysicalToConceptual().transform(verified), verified
96
96
  else:
97
97
  # Bug in the code
98
98
  raise ValueError(f"Invalid output from importer: {type(verified)}")
@@ -100,24 +100,24 @@ class NeatRulesStore:
100
100
  def _graph_import_verify_convert(
101
101
  self,
102
102
  extractor: KnowledgeGraphExtractor,
103
- ) -> tuple[ConceptualDataModel, DMSRules | None]:
104
- info = extractor.get_information_rules()
105
- dms: DMSRules | None = None
103
+ ) -> tuple[ConceptualDataModel, PhysicalDataModel | None]:
104
+ conceptual = extractor.get_conceptual_data_model()
105
+ physical: PhysicalDataModel | None = None
106
106
  if isinstance(extractor, DMSGraphExtractor):
107
- dms = extractor.get_dms_rules()
108
- return info, dms
107
+ physical = extractor.get_physical_data_model()
108
+ return conceptual, physical
109
109
 
110
110
  def _manual_transform(
111
111
  self, importer: BaseImporter, validate: bool = True, client: NeatClient | None = None
112
112
  ) -> IssueList:
113
113
  result, issue_list, start, end = self._do_activity(
114
- partial(self._rules_import_verify_convert, importer, validate, client)
114
+ partial(self._data_model_import_verify_convert, importer, validate, client)
115
115
  )
116
116
 
117
117
  if not result:
118
118
  return issue_list
119
119
 
120
- info, dms = result
120
+ conceptual, physical = result
121
121
  last_change = self.provenance[-1]
122
122
 
123
123
  outside_agent = EXTERNAL_AGENT
@@ -128,67 +128,67 @@ class NeatRulesStore:
128
128
  used=last_change.target_entity,
129
129
  )
130
130
 
131
- # Case 1: Source of imported rules is not known
131
+ # Case 1: Source of imported data model is not known
132
132
  if not (source_id := self._get_source_id(result)):
133
133
  raise NeatValueError(
134
- "The source of the imported rules is unknown."
134
+ "The source of the imported data model is unknown."
135
135
  " Import will be skipped. Start a new NEAT session and import the data model there."
136
136
  )
137
137
 
138
- # Case 2: Source of imported rules not in rules_store
138
+ # Case 2: Source of imported data model not in data_model_store
139
139
  if not (source_entity := self.provenance.target_entity(source_id)) or not isinstance(
140
- source_entity, RulesEntity
140
+ source_entity, DataModelEntity
141
141
  ):
142
142
  raise NeatValueError(
143
- "The source of the imported rules is not in the provenance."
143
+ "The source of the imported data model is not in the provenance."
144
144
  " Import will be skipped. Start a new NEAT session and import the data model there."
145
145
  )
146
146
 
147
147
  # Case 3: Source is not the latest source entity in the provenance change
148
148
  if source_entity.id_ != last_change.target_entity.id_:
149
149
  raise NeatValueError(
150
- "Imported rules are detached from the provenance chain."
150
+ "Imported data model is detached from the provenance chain."
151
151
  " Import will be skipped. Start a new NEAT session and import the data model there."
152
152
  )
153
153
 
154
154
  # Case 4: Provenance is already at the physical state of the data model, going back to logical not possible
155
- if not dms and source_entity.dms:
155
+ if not physical and source_entity.physical:
156
156
  raise NeatValueError(
157
- "Rules are already in physical state, import of logical model not possible."
157
+ "Data model is already in physical state, import of conceptual model not possible."
158
158
  " Import will be skipped. Start a new NEAT session and import the data model there."
159
159
  )
160
160
 
161
- # modification took place on information rules
162
- if not dms and not source_entity.dms:
163
- outside_target_entity = RulesEntity(
161
+ # modification took place on conceptual data model
162
+ if not physical and not source_entity.physical:
163
+ outside_target_entity = DataModelEntity(
164
164
  was_attributed_to=outside_agent,
165
165
  was_generated_by=outside_activity,
166
- information=info,
167
- dms=dms,
166
+ conceptual=conceptual,
167
+ physical=physical,
168
168
  issues=issue_list,
169
- id_=self._create_id(info, dms),
169
+ id_=self._create_id(conceptual, physical),
170
170
  )
171
171
 
172
- # modification took place on dms rules, keep latest information rules
173
- elif dms and source_entity.dms:
174
- outside_target_entity = RulesEntity(
172
+ # modification took place on physical data model, keep latest conceptual data model
173
+ elif physical and source_entity.physical:
174
+ outside_target_entity = DataModelEntity(
175
175
  was_attributed_to=outside_agent,
176
176
  was_generated_by=outside_activity,
177
- information=last_change.target_entity.information,
178
- dms=dms,
177
+ conceptual=last_change.target_entity.conceptual,
178
+ physical=physical,
179
179
  issues=issue_list,
180
- id_=self._create_id(info, dms),
180
+ id_=self._create_id(conceptual, physical),
181
181
  )
182
182
 
183
183
  else:
184
- raise NeatValueError("Invalid state of rules for manual transformation")
184
+ raise NeatValueError("Invalid state of data model for manual transformation")
185
185
 
186
186
  outside_change = Change(
187
187
  source_entity=last_change.target_entity,
188
188
  agent=outside_agent,
189
189
  activity=outside_activity,
190
190
  target_entity=outside_target_entity,
191
- description="Manual transformation of rules outside of NEAT",
191
+ description="Manual transformation of data model outside of NEAT",
192
192
  )
193
193
 
194
194
  self._last_issues = issue_list
@@ -203,7 +203,7 @@ class NeatRulesStore:
203
203
  else:
204
204
  return self.do_activity(partial(self._graph_import_verify_convert, extractor), extractor)
205
205
 
206
- def import_rules(
206
+ def import_data_model(
207
207
  self,
208
208
  importer: BaseImporter,
209
209
  validate: bool = True,
@@ -212,15 +212,15 @@ class NeatRulesStore:
212
212
  ) -> IssueList:
213
213
  if self.empty:
214
214
  return self.do_activity(
215
- partial(self._rules_import_verify_convert, importer, validate, client),
215
+ partial(self._data_model_import_verify_convert, importer, validate, client),
216
216
  importer,
217
217
  )
218
218
  elif enable_manual_edit:
219
219
  return self._manual_transform(importer, validate, client)
220
220
  else:
221
- raise NeatValueError("Re-importing rules in the rules store is not allowed.")
221
+ raise NeatValueError("Re-importing data model in the data model store is not allowed.")
222
222
 
223
- def transform(self, *transformer: VerifiedRulesTransformer) -> IssueList:
223
+ def transform(self, *transformer: VerifiedDataModelTransformer) -> IssueList:
224
224
  if not self.provenance:
225
225
  raise EmptyStore()
226
226
 
@@ -228,26 +228,26 @@ class NeatRulesStore:
228
228
  for agent_tool in transformer:
229
229
 
230
230
  def action(
231
- transformer_item: VerifiedRulesTransformer = agent_tool,
232
- ) -> tuple[ConceptualDataModel, DMSRules | None]:
231
+ transformer_item: VerifiedDataModelTransformer = agent_tool,
232
+ ) -> tuple[ConceptualDataModel, PhysicalDataModel | None]:
233
233
  last_change = self.provenance[-1]
234
234
  source_entity = last_change.target_entity
235
235
  transformer_input = self._get_transformer_input(source_entity, transformer_item)
236
236
  transformer_output = transformer_item.transform(transformer_input)
237
237
  if isinstance(transformer_output, ConceptualDataModel):
238
238
  return transformer_output, None
239
- return last_change.target_entity.information, transformer_output
239
+ return last_change.target_entity.conceptual, transformer_output
240
240
 
241
241
  issues = self.do_activity(action, agent_tool)
242
242
  all_issues.extend(issues)
243
243
 
244
244
  return all_issues
245
245
 
246
- def export(self, exporter: BaseExporter[T_VerifiedRules, T_Export]) -> T_Export:
246
+ def export(self, exporter: BaseExporter[T_VerifiedDataModel, T_Export]) -> T_Export:
247
247
  return self._export_activity(exporter.export, exporter, DEFAULT_NAMESPACE["export-result"])
248
248
 
249
249
  def export_to_file(self, exporter: BaseExporter, path: Path) -> None:
250
- def export_action(input_: VerifiedRules) -> Path:
250
+ def export_action(input_: VerifiedDataModel) -> Path:
251
251
  exporter.export_to_file(input_, path)
252
252
  return path
253
253
 
@@ -260,8 +260,8 @@ class NeatRulesStore:
260
260
 
261
261
  def do_activity(
262
262
  self,
263
- action: Callable[[], tuple[ConceptualDataModel, DMSRules | None]],
264
- agent_tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
263
+ action: Callable[[], tuple[ConceptualDataModel, PhysicalDataModel | None]],
264
+ agent_tool: (BaseImporter | VerifiedDataModelTransformer | KnowledgeGraphExtractor),
265
265
  ) -> IssueList:
266
266
  result, issue_list, start, end = self._do_activity(action)
267
267
  self._last_issues = issue_list
@@ -272,8 +272,8 @@ class NeatRulesStore:
272
272
 
273
273
  def _update_provenance(
274
274
  self,
275
- agent_tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
276
- result: tuple[ConceptualDataModel, DMSRules | None],
275
+ agent_tool: (BaseImporter | VerifiedDataModelTransformer | KnowledgeGraphExtractor),
276
+ result: tuple[ConceptualDataModel, PhysicalDataModel | None],
277
277
  issue_list: IssueList,
278
278
  activity_start: datetime,
279
279
  activity_end: datetime,
@@ -288,7 +288,7 @@ class NeatRulesStore:
288
288
  source_entity = self.provenance[-1].target_entity
289
289
 
290
290
  # setting the rest of provenance components
291
- info, dms = result
291
+ conceptual, physical = result
292
292
  agent = agent_tool.agent
293
293
  activity = Activity(
294
294
  was_associated_with=agent,
@@ -297,14 +297,14 @@ class NeatRulesStore:
297
297
  used=source_entity,
298
298
  )
299
299
 
300
- target_entity = RulesEntity(
300
+ target_entity = DataModelEntity(
301
301
  was_attributed_to=agent,
302
302
  was_generated_by=activity,
303
- information=info,
304
- dms=dms,
303
+ conceptual=conceptual,
304
+ physical=physical,
305
305
  issues=issue_list,
306
306
  # here id can be bumped in case id already exists
307
- id_=self._create_id(info, dms),
307
+ id_=self._create_id(conceptual, physical),
308
308
  )
309
309
  change = Change(
310
310
  agent=agent,
@@ -318,11 +318,16 @@ class NeatRulesStore:
318
318
 
319
319
  def _do_activity(
320
320
  self,
321
- action: Callable[[], tuple[ConceptualDataModel, DMSRules | None]],
322
- ) -> tuple[tuple[ConceptualDataModel, DMSRules | None], IssueList, datetime, datetime]:
321
+ action: Callable[[], tuple[ConceptualDataModel, PhysicalDataModel | None]],
322
+ ) -> tuple[
323
+ tuple[ConceptualDataModel, PhysicalDataModel | None],
324
+ IssueList,
325
+ datetime,
326
+ datetime,
327
+ ]:
323
328
  """This private method is used to execute an activity and return the result and issues."""
324
329
  start = datetime.now(timezone.utc)
325
- result: tuple[ConceptualDataModel, DMSRules | None] | None = None
330
+ result: tuple[ConceptualDataModel, PhysicalDataModel | None] | None = None
326
331
  with catch_issues() as issue_list:
327
332
  result = action()
328
333
  end = datetime.now(timezone.utc)
@@ -335,14 +340,14 @@ class NeatRulesStore:
335
340
  source_entity = last_change.target_entity
336
341
  expected_types = exporter.source_types()
337
342
 
338
- if source_entity.dms is not None and isinstance(source_entity.dms, expected_types):
339
- input_ = cast(VerifiedRules, source_entity.dms).model_copy(deep=True)
340
- elif isinstance(source_entity.information, expected_types):
341
- input_ = cast(VerifiedRules, source_entity.information).model_copy(deep=True)
343
+ if source_entity.physical is not None and isinstance(source_entity.physical, expected_types):
344
+ input_ = cast(VerifiedDataModel, source_entity.physical).model_copy(deep=True)
345
+ elif isinstance(source_entity.conceptual, expected_types):
346
+ input_ = cast(VerifiedDataModel, source_entity.conceptual).model_copy(deep=True)
342
347
  else:
343
348
  available: list[type] = [ConceptualDataModel]
344
- if source_entity.dms is not None:
345
- available.append(DMSRules)
349
+ if source_entity.physical is not None:
350
+ available.append(PhysicalDataModel)
346
351
  raise InvalidActivityInput(expected=expected_types, have=tuple(available))
347
352
 
348
353
  # need to write source prior the export
@@ -389,33 +394,35 @@ class NeatRulesStore:
389
394
 
390
395
  @staticmethod
391
396
  def _get_transformer_input(
392
- source_entity: RulesEntity, transformer: VerifiedRulesTransformer
393
- ) -> ConceptualDataModel | DMSRules:
394
- # Case 1: We only have information rules
395
- if source_entity.dms is None:
396
- if transformer.is_valid_input(source_entity.information):
397
- return source_entity.information
398
- raise InvalidActivityInput(expected=(DMSRules,), have=(ConceptualDataModel,))
399
- # Case 2: We have both information and dms rules and the transformer is compatible with dms rules
400
- elif isinstance(source_entity.dms, DMSRules) and transformer.is_valid_input(source_entity.dms):
401
- return source_entity.dms
402
- # Case 3: We have both information and dms rules and the transformer is compatible with information rules
403
- raise InvalidActivityInput(expected=(ConceptualDataModel,), have=(DMSRules,))
404
-
405
- def _get_source_id(self, result: tuple[ConceptualDataModel, DMSRules | None]) -> rdflib.URIRef | None:
397
+ source_entity: DataModelEntity, transformer: VerifiedDataModelTransformer
398
+ ) -> ConceptualDataModel | PhysicalDataModel:
399
+ # Case 1: We only have conceptual data model
400
+ if source_entity.physical is None:
401
+ if transformer.is_valid_input(source_entity.conceptual):
402
+ return source_entity.conceptual
403
+ raise InvalidActivityInput(expected=(PhysicalDataModel,), have=(ConceptualDataModel,))
404
+ # Case 2: We have both data model levels and the transformer is compatible with physical data model
405
+ elif isinstance(source_entity.physical, PhysicalDataModel) and transformer.is_valid_input(
406
+ source_entity.physical
407
+ ):
408
+ return source_entity.physical
409
+ # Case 3: We have both data model levels and the transformer is compatible with conceptual data model
410
+ raise InvalidActivityInput(expected=(ConceptualDataModel,), have=(PhysicalDataModel,))
411
+
412
+ def _get_source_id(self, result: tuple[ConceptualDataModel, PhysicalDataModel | None]) -> rdflib.URIRef | None:
406
413
  """Return the source of the result.
407
414
 
408
415
  !!! note
409
- This method prioritizes the source_id of the DMS rules
416
+ This method prioritizes the source_id of the physical data model
410
417
  """
411
- info, dms = result
412
- return dms.metadata.source_id if dms else info.metadata.source_id
418
+ conceptual, physical = result
419
+ return physical.metadata.source_id if physical else conceptual.metadata.source_id
413
420
 
414
- def _create_id(self, info: ConceptualDataModel, dms: DMSRules | None) -> rdflib.URIRef:
415
- if dms is None:
416
- identifier = info.metadata.identifier
421
+ def _create_id(self, conceptual: ConceptualDataModel, physical: PhysicalDataModel | None) -> rdflib.URIRef:
422
+ if physical is None:
423
+ identifier = conceptual.metadata.identifier
417
424
  else:
418
- identifier = dms.metadata.identifier
425
+ identifier = physical.metadata.identifier
419
426
 
420
427
  # Here we check if the identifier is already in the iteration dictionary
421
428
  # to track specific changes to the same entity, if it is we increment the iteration
@@ -429,39 +436,39 @@ class NeatRulesStore:
429
436
  return identifier + f"/Iteration_{self._iteration_by_id[identifier]}"
430
437
 
431
438
  @property
432
- def try_get_last_dms_rules(self) -> DMSRules | None:
439
+ def try_get_last_physical_data_model(self) -> PhysicalDataModel | None:
433
440
  if not self.provenance:
434
441
  return None
435
- if self.provenance[-1].target_entity.dms is None:
442
+ if self.provenance[-1].target_entity.physical is None:
436
443
  return None
437
- return self.provenance[-1].target_entity.dms
444
+ return self.provenance[-1].target_entity.physical
438
445
 
439
446
  @property
440
- def try_get_last_information_rules(self) -> ConceptualDataModel | None:
447
+ def try_get_last_conceptual_data_model(self) -> ConceptualDataModel | None:
441
448
  if not self.provenance:
442
449
  return None
443
- return self.provenance[-1].target_entity.information
450
+ return self.provenance[-1].target_entity.conceptual
444
451
 
445
452
  @property
446
- def last_verified_dms_rules(self) -> DMSRules:
453
+ def last_verified_physical_data_model(self) -> PhysicalDataModel:
447
454
  if not self.provenance:
448
455
  raise EmptyStore()
449
- if self.provenance[-1].target_entity.dms is None:
450
- raise NeatValueError("No verified DMS rules found in the provenance.")
451
- return self.provenance[-1].target_entity.dms
456
+ if self.provenance[-1].target_entity.physical is None:
457
+ raise NeatValueError("No verified physical data model found in the provenance.")
458
+ return self.provenance[-1].target_entity.physical
452
459
 
453
460
  @property
454
- def last_verified_information_rules(self) -> ConceptualDataModel:
461
+ def last_verified_conceptual_data_model(self) -> ConceptualDataModel:
455
462
  if not self.provenance:
456
463
  raise EmptyStore()
457
- return self.provenance[-1].target_entity.information
464
+ return self.provenance[-1].target_entity.conceptual
458
465
 
459
466
  @property
460
- def last_verified_rules(self) -> ConceptualDataModel | DMSRules | None:
467
+ def last_verified_data_model(self) -> ConceptualDataModel | PhysicalDataModel | None:
461
468
  if not self.provenance:
462
469
  return None
463
470
  last_entity = self.provenance[-1].target_entity
464
- return last_entity.dms or last_entity.information
471
+ return last_entity.physical or last_entity.conceptual
465
472
 
466
473
  @property
467
474
  def last_issues(self) -> IssueList | None:
@@ -34,13 +34,12 @@ else:
34
34
  from typing import Self
35
35
 
36
36
 
37
- class NeatGraphStore:
38
- """NeatGraphStore is a class that stores the graph and provides methods to read/write data it contains
37
+ class NeatInstanceStore:
38
+ """NeatInstanceStore is a class that stores instances as triples and provides methods to read/write data it contains
39
39
 
40
40
 
41
41
  Args:
42
- graph : Instance of rdflib.Graph class for graph storage
43
- rules:
42
+ dataset : Instance of rdflib.Dataset class for instance storage
44
43
 
45
44
  !!! note "Dataset"
46
45
  The store leverages a RDF dataset which is defined as a collection of RDF graphs
@@ -10,9 +10,9 @@ where in case of neat when dealing with instances we have:
10
10
 
11
11
  and in case of data models we have:
12
12
 
13
- * Agent: Rules importers, exporters, transformers, etc.
13
+ * Agent: Data Model importers, exporters, transformers, etc.
14
14
  * Activity: convert, verify, etc.
15
- * Entity: data model (aka Rules)
15
+ * Entity: data model (aka Data Model)
16
16
 
17
17
  """
18
18
 
@@ -3,7 +3,7 @@
3
3
  from dataclasses import dataclass
4
4
 
5
5
  from cognite.neat.core._data_model.importers import BaseImporter
6
- from cognite.neat.core._data_model.transformers import VerifiedRulesTransformer
6
+ from cognite.neat.core._data_model.transformers import VerifiedDataModelTransformer
7
7
  from cognite.neat.core._instances.extractors import KnowledgeGraphExtractor
8
8
  from cognite.neat.core._issues import IssueList
9
9
 
@@ -24,7 +24,7 @@ class ActivityFailed(NeatStoreError):
24
24
  self,
25
25
  activity: Activity,
26
26
  issue_list: IssueList,
27
- tool: BaseImporter | VerifiedRulesTransformer | KnowledgeGraphExtractor,
27
+ tool: BaseImporter | VerifiedDataModelTransformer | KnowledgeGraphExtractor,
28
28
  ) -> None:
29
29
  self.activity = activity
30
30
  self.issue_list = issue_list
@@ -1,4 +1,5 @@
1
1
  import re
2
+ import urllib.parse
2
3
  from collections.abc import Iterable
3
4
  from typing import Any, Literal, TypeAlias, overload
4
5
 
@@ -296,3 +297,16 @@ def uri_display_name(thing: URIRef) -> str:
296
297
  elif "http://purl.org/cognite/neat/data-model/" in thing:
297
298
  return "NEAT(" + ",".join(thing.replace("http://purl.org/cognite/neat/data-model/", "").split("/")) + ")"
298
299
  return remove_namespace_from_uri(thing)
300
+
301
+
302
+ def uri_to_cdf_id(uri: URIRef) -> str:
303
+ """Convert a URI to a user-friendly string.
304
+ Removing the namespace and unquoting the URI.
305
+ If the namespace is a CDF space, then it will be included in the string as space:externalId.
306
+ """
307
+ namespace, entity_id = split_uri(uri)
308
+ output_str = urllib.parse.unquote(entity_id)
309
+
310
+ if space := namespace_as_space(namespace):
311
+ output_str = f"{space}:{output_str}"
312
+ return output_str
@@ -236,7 +236,7 @@ class NamingStandardization:
236
236
  _start_letter_pattern = re.compile(r"^[a-zA-Z]")
237
237
 
238
238
  @classmethod
239
- def standardize_class_str(cls, raw: str) -> str:
239
+ def standardize_concept_str(cls, raw: str) -> str:
240
240
  clean = cls._clean_string(raw)
241
241
  if not cls._start_letter_pattern.match(clean):
242
242
  # Underscore ensure that 'Class' it treated as a separate word