cognite-neat 0.121.0__py3-none-any.whl → 0.121.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (150) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/core/_client/_api/statistics.py +91 -0
  3. cognite/neat/core/_client/_api_client.py +2 -0
  4. cognite/neat/core/_client/data_classes/statistics.py +125 -0
  5. cognite/neat/core/_client/testing.py +4 -0
  6. cognite/neat/core/_constants.py +6 -7
  7. cognite/neat/core/{_rules → _data_model}/_constants.py +25 -18
  8. cognite/neat/core/_data_model/_shared.py +59 -0
  9. cognite/neat/core/_data_model/analysis/__init__.py +3 -0
  10. cognite/neat/core/{_rules → _data_model}/analysis/_base.py +202 -195
  11. cognite/neat/core/{_rules → _data_model}/catalog/__init__.py +1 -1
  12. cognite/neat/core/{_rules → _data_model}/exporters/__init__.py +5 -5
  13. cognite/neat/core/{_rules → _data_model}/exporters/_base.py +10 -8
  14. cognite/neat/core/{_rules/exporters/_rules2dms.py → _data_model/exporters/_data_model2dms.py} +22 -18
  15. cognite/neat/core/{_rules/exporters/_rules2excel.py → _data_model/exporters/_data_model2excel.py} +61 -56
  16. cognite/neat/core/{_rules/exporters/_rules2instance_template.py → _data_model/exporters/_data_model2instance_template.py} +11 -9
  17. cognite/neat/core/{_rules/exporters/_rules2ontology.py → _data_model/exporters/_data_model2ontology.py} +64 -61
  18. cognite/neat/core/{_rules/exporters/_rules2yaml.py → _data_model/exporters/_data_model2yaml.py} +21 -18
  19. cognite/neat/core/{_rules → _data_model}/importers/__init__.py +6 -8
  20. cognite/neat/core/{_rules → _data_model}/importers/_base.py +8 -6
  21. cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
  22. cognite/neat/core/{_rules/importers/_yaml2rules.py → _data_model/importers/_dict2data_model.py} +41 -21
  23. cognite/neat/core/{_rules/importers/_dms2rules.py → _data_model/importers/_dms2data_model.py} +79 -66
  24. cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_converter.py +41 -41
  25. cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/dtdl_importer.py +16 -16
  26. cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/spec.py +3 -3
  27. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_base.py +18 -16
  28. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_imf2rules.py +17 -17
  29. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_inference2rules.py +50 -50
  30. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_owl2rules.py +14 -14
  31. cognite/neat/core/{_rules → _data_model}/importers/_rdf/_shared.py +25 -25
  32. cognite/neat/core/{_rules/importers/_spreadsheet2rules.py → _data_model/importers/_spreadsheet2data_model.py} +69 -38
  33. cognite/neat/core/_data_model/models/__init__.py +36 -0
  34. cognite/neat/core/{_rules/models/_base_input.py → _data_model/models/_base_unverified.py} +12 -12
  35. cognite/neat/core/{_rules/models/_base_rules.py → _data_model/models/_base_verified.py} +13 -13
  36. cognite/neat/core/{_rules → _data_model}/models/_types.py +13 -13
  37. cognite/neat/core/_data_model/models/conceptual/__init__.py +25 -0
  38. cognite/neat/core/{_rules/models/information/_rules_input.py → _data_model/models/conceptual/_unverified.py} +46 -43
  39. cognite/neat/core/{_rules/models/information → _data_model/models/conceptual}/_validation.py +93 -79
  40. cognite/neat/core/{_rules/models/information/_rules.py → _data_model/models/conceptual/_verified.py} +83 -83
  41. cognite/neat/core/{_rules → _data_model}/models/data_types.py +4 -4
  42. cognite/neat/core/{_rules → _data_model}/models/entities/__init__.py +8 -8
  43. cognite/neat/core/{_rules → _data_model}/models/entities/_loaders.py +12 -11
  44. cognite/neat/core/{_rules → _data_model}/models/entities/_multi_value.py +7 -7
  45. cognite/neat/core/{_rules → _data_model}/models/entities/_single_value.py +45 -39
  46. cognite/neat/core/{_rules → _data_model}/models/entities/_types.py +9 -3
  47. cognite/neat/core/{_rules → _data_model}/models/entities/_wrapped.py +3 -3
  48. cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.py +12 -9
  49. cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
  50. cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_exporter.py +83 -64
  51. cognite/neat/core/{_rules/models/dms/_rules_input.py → _data_model/models/physical/_unverified.py} +56 -44
  52. cognite/neat/core/{_rules/models/dms → _data_model/models/physical}/_validation.py +20 -17
  53. cognite/neat/core/{_rules/models/dms/_rules.py → _data_model/models/physical/_verified.py} +79 -71
  54. cognite/neat/core/{_rules → _data_model}/transformers/__init__.py +27 -23
  55. cognite/neat/core/{_rules → _data_model}/transformers/_base.py +29 -19
  56. cognite/neat/core/{_rules → _data_model}/transformers/_converters.py +758 -659
  57. cognite/neat/core/{_rules → _data_model}/transformers/_mapping.py +79 -60
  58. cognite/neat/core/_data_model/transformers/_verification.py +120 -0
  59. cognite/neat/core/{_graph → _instances}/extractors/_base.py +2 -2
  60. cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_base.py +1 -1
  61. cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_classic.py +17 -11
  62. cognite/neat/core/{_graph → _instances}/extractors/_dms_graph.py +47 -39
  63. cognite/neat/core/{_graph → _instances}/extractors/_mock_graph_generator.py +102 -99
  64. cognite/neat/core/{_graph → _instances}/extractors/_rdf_file.py +2 -2
  65. cognite/neat/core/{_graph → _instances}/loaders/_base.py +2 -2
  66. cognite/neat/core/{_graph → _instances}/loaders/_rdf2dms.py +16 -14
  67. cognite/neat/core/{_graph → _instances}/transformers/_base.py +7 -4
  68. cognite/neat/core/{_graph → _instances}/transformers/_classic_cdf.py +1 -1
  69. cognite/neat/core/{_graph → _instances}/transformers/_value_type.py +2 -6
  70. cognite/neat/core/_issues/_base.py +4 -4
  71. cognite/neat/core/_issues/errors/__init__.py +2 -2
  72. cognite/neat/core/_issues/errors/_wrapper.py +2 -2
  73. cognite/neat/core/_issues/warnings/__init__.py +2 -0
  74. cognite/neat/core/_issues/warnings/_models.py +4 -4
  75. cognite/neat/core/_issues/warnings/_properties.py +7 -0
  76. cognite/neat/core/_store/__init__.py +3 -3
  77. cognite/neat/core/_store/{_rules_store.py → _data_model.py} +128 -121
  78. cognite/neat/core/_store/{_graph_store.py → _instance.py} +7 -8
  79. cognite/neat/core/_store/_provenance.py +2 -2
  80. cognite/neat/core/_store/exceptions.py +4 -4
  81. cognite/neat/core/_utils/rdf_.py +14 -0
  82. cognite/neat/core/_utils/spreadsheet.py +1 -1
  83. cognite/neat/core/_utils/text.py +2 -2
  84. cognite/neat/session/_base.py +29 -25
  85. cognite/neat/session/_drop.py +3 -3
  86. cognite/neat/session/_fix.py +2 -2
  87. cognite/neat/session/_inspect.py +5 -5
  88. cognite/neat/session/_mapping.py +11 -9
  89. cognite/neat/session/_prepare.py +4 -4
  90. cognite/neat/session/_read.py +15 -15
  91. cognite/neat/session/_set.py +5 -5
  92. cognite/neat/session/_show.py +11 -11
  93. cognite/neat/session/_state.py +17 -17
  94. cognite/neat/session/_subset.py +14 -11
  95. cognite/neat/session/_template.py +19 -19
  96. cognite/neat/session/_to.py +21 -21
  97. cognite/neat/session/_wizard.py +1 -1
  98. {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/METADATA +1 -1
  99. cognite_neat-0.121.2.dist-info/RECORD +189 -0
  100. cognite/neat/core/_rules/_shared.py +0 -43
  101. cognite/neat/core/_rules/analysis/__init__.py +0 -3
  102. cognite/neat/core/_rules/exporters/_validation.py +0 -14
  103. cognite/neat/core/_rules/models/__init__.py +0 -34
  104. cognite/neat/core/_rules/models/dms/__init__.py +0 -32
  105. cognite/neat/core/_rules/models/information/__init__.py +0 -20
  106. cognite/neat/core/_rules/transformers/_verification.py +0 -111
  107. cognite_neat-0.121.0.dist-info/RECORD +0 -187
  108. /cognite/neat/core/{_graph → _data_model}/__init__.py +0 -0
  109. /cognite/neat/core/{_rules → _data_model}/catalog/classic_model.xlsx +0 -0
  110. /cognite/neat/core/{_rules/catalog/info-rules-imf.xlsx → _data_model/catalog/conceptual-imf-data-model.xlsx} +0 -0
  111. /cognite/neat/core/{_rules → _data_model}/catalog/hello_world_pump.xlsx +0 -0
  112. /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/__init__.py +0 -0
  113. /cognite/neat/core/{_rules/importers/_dtdl2rules → _data_model/importers/_dtdl2data_model}/_unit_lookup.py +0 -0
  114. /cognite/neat/core/{_rules → _data_model}/importers/_rdf/__init__.py +0 -0
  115. /cognite/neat/core/{_rules → _data_model}/models/entities/_constants.py +0 -0
  116. /cognite/neat/core/{_rules → _data_model}/models/mapping/__init__.py +0 -0
  117. /cognite/neat/core/{_rules → _data_model}/models/mapping/_classic2core.yaml +0 -0
  118. /cognite/neat/core/{_graph/extractors/_classic_cdf → _instances}/__init__.py +0 -0
  119. /cognite/neat/core/{_graph → _instances}/_shared.py +0 -0
  120. /cognite/neat/core/{_graph → _instances}/_tracking/__init__.py +0 -0
  121. /cognite/neat/core/{_graph → _instances}/_tracking/base.py +0 -0
  122. /cognite/neat/core/{_graph → _instances}/_tracking/log.py +0 -0
  123. /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -0
  124. /cognite/neat/core/{_graph → _instances}/examples/Knowledge-Graph-Nordic44.xml +0 -0
  125. /cognite/neat/core/{_graph → _instances}/examples/__init__.py +0 -0
  126. /cognite/neat/core/{_graph → _instances}/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  127. /cognite/neat/core/{_graph → _instances}/extractors/__init__.py +0 -0
  128. /cognite/neat/core/{_rules → _instances/extractors/_classic_cdf}/__init__.py +0 -0
  129. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_assets.py +0 -0
  130. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_data_sets.py +0 -0
  131. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_events.py +0 -0
  132. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_files.py +0 -0
  133. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_labels.py +0 -0
  134. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_relationships.py +0 -0
  135. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_sequences.py +0 -0
  136. /cognite/neat/core/{_graph → _instances}/extractors/_classic_cdf/_timeseries.py +0 -0
  137. /cognite/neat/core/{_graph → _instances}/extractors/_dict.py +0 -0
  138. /cognite/neat/core/{_graph → _instances}/extractors/_dms.py +0 -0
  139. /cognite/neat/core/{_graph → _instances}/extractors/_raw.py +0 -0
  140. /cognite/neat/core/{_graph → _instances}/loaders/__init__.py +0 -0
  141. /cognite/neat/core/{_graph → _instances}/queries/__init__.py +0 -0
  142. /cognite/neat/core/{_graph → _instances}/queries/_base.py +0 -0
  143. /cognite/neat/core/{_graph → _instances}/queries/_queries.py +0 -0
  144. /cognite/neat/core/{_graph → _instances}/queries/_select.py +0 -0
  145. /cognite/neat/core/{_graph → _instances}/queries/_update.py +0 -0
  146. /cognite/neat/core/{_graph → _instances}/transformers/__init__.py +0 -0
  147. /cognite/neat/core/{_graph → _instances}/transformers/_prune_graph.py +0 -0
  148. /cognite/neat/core/{_graph → _instances}/transformers/_rdfpath.py +0 -0
  149. {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/WHEEL +0 -0
  150. {cognite_neat-0.121.0.dist-info → cognite_neat-0.121.2.dist-info}/licenses/LICENSE +0 -0
@@ -6,25 +6,29 @@ from typing import Any, ClassVar, Literal
6
6
  from cognite.client import data_modeling as dm
7
7
 
8
8
  from cognite.neat.core._client import NeatClient
9
+ from cognite.neat.core._data_model.models import PhysicalDataModel, SheetList
10
+ from cognite.neat.core._data_model.models.data_types import Enum
11
+ from cognite.neat.core._data_model.models.entities import (
12
+ ConceptEntity,
13
+ ContainerEntity,
14
+ ViewEntity,
15
+ )
16
+ from cognite.neat.core._data_model.models.physical import (
17
+ PhysicalContainer,
18
+ PhysicalEnum,
19
+ PhysicalProperty,
20
+ )
9
21
  from cognite.neat.core._issues.errors import (
10
22
  CDFMissingClientError,
11
23
  NeatValueError,
12
24
  ResourceNotFoundError,
13
25
  )
14
26
  from cognite.neat.core._issues.warnings import PropertyOverwritingWarning
15
- from cognite.neat.core._rules.models import DMSRules, SheetList
16
- from cognite.neat.core._rules.models.data_types import Enum
17
- from cognite.neat.core._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
18
- from cognite.neat.core._rules.models.entities import (
19
- ClassEntity,
20
- ContainerEntity,
21
- ViewEntity,
22
- )
23
27
 
24
- from ._base import VerifiedRulesTransformer
28
+ from ._base import VerifiedDataModelTransformer
25
29
 
26
30
 
27
- class MapOntoTransformers(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
31
+ class MapOntoTransformers(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel], ABC):
28
32
  """Base class for transformers that map one rule onto another."""
29
33
 
30
34
  ...
@@ -33,7 +37,7 @@ class MapOntoTransformers(VerifiedRulesTransformer[DMSRules, DMSRules], ABC):
33
37
  class MapOneToOne(MapOntoTransformers):
34
38
  """Takes transform data models and makes it into an extension of the reference data model.
35
39
 
36
- Note this transformer mutates the input rules.
40
+ Note this transformer mutates the input data model.
37
41
 
38
42
  The argument view_extension_mapping is a dictionary where the keys are views of this data model,
39
43
  and each value is the view of the reference data model that the view should extend. For example:
@@ -55,14 +59,17 @@ class MapOneToOne(MapOntoTransformers):
55
59
  """
56
60
 
57
61
  def __init__(
58
- self, reference: DMSRules, view_extension_mapping: dict[str, str], default_extension: str | None = None
62
+ self,
63
+ reference: PhysicalDataModel,
64
+ view_extension_mapping: dict[str, str],
65
+ default_extension: str | None = None,
59
66
  ) -> None:
60
67
  self.reference = reference
61
68
  self.view_extension_mapping = view_extension_mapping
62
69
  self.default_extension = default_extension
63
70
 
64
- def transform(self, rules: DMSRules) -> DMSRules:
65
- solution: DMSRules = rules
71
+ def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
72
+ solution: PhysicalDataModel = data_model
66
73
  view_by_external_id = {view.view.external_id: view for view in solution.views}
67
74
  ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
68
75
 
@@ -73,11 +80,11 @@ class MapOneToOne(MapOntoTransformers):
73
80
  if self.default_extension and self.default_extension not in ref_view_by_external_id:
74
81
  raise ValueError(f"Default extension view not in the reference data model {self.default_extension}")
75
82
 
76
- properties_by_view_external_id: dict[str, dict[str, DMSProperty]] = defaultdict(dict)
83
+ properties_by_view_external_id: dict[str, dict[str, PhysicalProperty]] = defaultdict(dict)
77
84
  for prop in solution.properties:
78
85
  properties_by_view_external_id[prop.view.external_id][prop.view_property] = prop
79
86
 
80
- ref_properties_by_view_external_id: dict[str, dict[str, DMSProperty]] = defaultdict(dict)
87
+ ref_properties_by_view_external_id: dict[str, dict[str, PhysicalProperty]] = defaultdict(dict)
81
88
  for prop in self.reference.properties:
82
89
  ref_properties_by_view_external_id[prop.view.external_id][prop.view_property] = prop
83
90
 
@@ -108,14 +115,15 @@ class MapOneToOne(MapOntoTransformers):
108
115
  return solution
109
116
 
110
117
 
111
- class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
118
+ class PhysicalDataModelMapper(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
112
119
  """Maps properties and classes using the given mapping.
113
120
 
114
121
  Args:
115
- mapping: The mapping to use represented as a DMSRules object.
122
+ mapping: The mapping to use represented as a physical data model object.
116
123
  data_type_conflict: How to handle data type conflicts. The default is "overwrite".
117
124
  A data type conflicts occurs when the data type of a property in the mapping is different from the
118
- data type of the property in the input rules. If "overwrite" the data type in the input rules is overwritten
125
+ data type of the property in the input data model. If "overwrite" the data type
126
+ in the input data model is overwritten
119
127
  with the data type in the mapping.
120
128
  """
121
129
 
@@ -123,37 +131,41 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
123
131
  ["connection", "value_type", "min_count", "immutable", "max_count", "default", "index", "constraint"]
124
132
  )
125
133
 
126
- def __init__(self, mapping: DMSRules, data_type_conflict: Literal["overwrite"] = "overwrite") -> None:
134
+ def __init__(
135
+ self,
136
+ mapping: PhysicalDataModel,
137
+ data_type_conflict: Literal["overwrite"] = "overwrite",
138
+ ) -> None:
127
139
  self.mapping = mapping
128
140
  self.data_type_conflict = data_type_conflict
129
141
 
130
- def transform(self, rules: DMSRules) -> DMSRules:
142
+ def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
131
143
  if self.data_type_conflict != "overwrite":
132
144
  raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
133
- input_rules = rules
134
- new_rules = input_rules.model_copy(deep=True)
145
+ input_data_model = data_model
146
+ new_data_model = input_data_model.model_copy(deep=True)
135
147
 
136
- views_by_external_id = {view.view.external_id: view for view in new_rules.views}
148
+ views_by_external_id = {view.view.external_id: view for view in new_data_model.views}
137
149
  new_views: set[ViewEntity] = set()
138
150
  for mapping_view in self.mapping.views:
139
151
  if existing_view := views_by_external_id.get(mapping_view.view.external_id):
140
152
  existing_view.implements = mapping_view.implements
141
153
  else:
142
- # We need to add all the views in the mapping that are not in the input rules.
143
- # This is to ensure that all ValueTypes are present in the resulting rules.
154
+ # We need to add all the views in the mapping that are not in the input data model.
155
+ # This is to ensure that all ValueTypes are present in the resulting data model.
144
156
  # For example, if a property is a direct relation to an Equipment view, we need to add
145
- # the Equipment view to the rules.
146
- new_rules.views.append(mapping_view)
157
+ # the Equipment view to the data model.
158
+ new_data_model.views.append(mapping_view)
147
159
  new_views.add(mapping_view.view)
148
160
 
149
161
  properties_by_view_property = {
150
- (prop.view.external_id, prop.view_property): prop for prop in new_rules.properties
162
+ (prop.view.external_id, prop.view_property): prop for prop in new_data_model.properties
151
163
  }
152
- existing_enum_collections = {item.collection for item in new_rules.enum or []}
153
- mapping_enums_by_collection: dict[ClassEntity, list[DMSEnum]] = defaultdict(list)
164
+ existing_enum_collections = {item.collection for item in new_data_model.enum or []}
165
+ mapping_enums_by_collection: dict[ConceptEntity, list[PhysicalEnum]] = defaultdict(list)
154
166
  for item in self.mapping.enum or []:
155
167
  mapping_enums_by_collection[item.collection].append(item)
156
- existing_containers = {container.container for container in new_rules.containers or []}
168
+ existing_containers = {container.container for container in new_data_model.containers or []}
157
169
  mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
158
170
  for mapping_prop in self.mapping.properties:
159
171
  if existing_prop := properties_by_view_property.get(
@@ -163,7 +175,10 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
163
175
  if conflicts and self.data_type_conflict == "overwrite":
164
176
  warnings.warn(
165
177
  PropertyOverwritingWarning(
166
- existing_prop.view.as_id(), "view", existing_prop.view_property, tuple(conflicts)
178
+ existing_prop.view.as_id(),
179
+ "view",
180
+ existing_prop.view_property,
181
+ tuple(conflicts),
167
182
  ),
168
183
  stacklevel=2,
169
184
  )
@@ -177,24 +192,24 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
177
192
  existing_prop.container = mapping_prop.container
178
193
  existing_prop.container_property = mapping_prop.container_property
179
194
  elif isinstance(mapping_prop.value_type, ViewEntity):
180
- # All connections must be included in the rules. This is to update the
195
+ # All connections must be included in the data model. This is to update the
181
196
  # ValueTypes of the implemented views.
182
- new_rules.properties.append(mapping_prop)
197
+ new_data_model.properties.append(mapping_prop)
183
198
  elif "guid" in mapping_prop.view_property.casefold():
184
199
  # All guid properties are included. Theses are necessary to get an appropriate
185
200
  # filter on the resulting view.
186
- new_rules.properties.append(mapping_prop)
201
+ new_data_model.properties.append(mapping_prop)
187
202
  else:
188
- # Skipping mapped properties that are not in the input rules.
203
+ # Skipping mapped properties that are not in the input data model.
189
204
  continue
190
205
 
191
206
  if (
192
207
  isinstance(mapping_prop.value_type, Enum)
193
208
  and mapping_prop.value_type.collection not in existing_enum_collections
194
209
  ):
195
- if not new_rules.enum:
196
- new_rules.enum = SheetList[DMSEnum]([])
197
- new_rules.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
210
+ if not new_data_model.enum:
211
+ new_data_model.enum = SheetList[PhysicalEnum]([])
212
+ new_data_model.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
198
213
 
199
214
  if (
200
215
  mapping_prop.container
@@ -202,13 +217,15 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
202
217
  and (new_container := mapping_containers_by_id.get(mapping_prop.container))
203
218
  ):
204
219
  # Mapping can include new containers for GUID properties
205
- if not new_rules.containers:
206
- new_rules.containers = SheetList[DMSContainer]([])
207
- new_rules.containers.append(new_container)
220
+ if not new_data_model.containers:
221
+ new_data_model.containers = SheetList[PhysicalContainer]([])
222
+ new_data_model.containers.append(new_container)
208
223
 
209
- return new_rules
224
+ return new_data_model
210
225
 
211
- def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
226
+ def _find_overwrites(
227
+ self, prop: PhysicalProperty, mapping_prop: PhysicalProperty
228
+ ) -> tuple[dict[str, Any], list[str]]:
212
229
  """Finds the properties that need to be overwritten and returns them.
213
230
 
214
231
  In addition, conflicting properties are returned. Note that overwriting properties that are
@@ -240,7 +257,7 @@ class RuleMapper(VerifiedRulesTransformer[DMSRules, DMSRules]):
240
257
  return f"Mapping to {self.mapping.metadata.as_data_model_id()!r}."
241
258
 
242
259
 
243
- class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
260
+ class AsParentPropertyId(VerifiedDataModelTransformer[PhysicalDataModel, PhysicalDataModel]):
244
261
  """Looks up all view properties that map to the same container property,
245
262
  and changes the child view property id to match the parent property id.
246
263
  """
@@ -248,34 +265,36 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
248
265
  def __init__(self, client: NeatClient | None = None) -> None:
249
266
  self._client = client
250
267
 
251
- def transform(self, rules: DMSRules) -> DMSRules:
252
- input_rules = rules
253
- new_rules = input_rules.model_copy(deep=True)
268
+ def transform(self, data_model: PhysicalDataModel) -> PhysicalDataModel:
269
+ input_data_model = data_model
270
+ new_data_model = input_data_model.model_copy(deep=True)
254
271
 
255
- path_by_view = self._inheritance_path_by_view(new_rules)
256
- view_by_container_property = self._view_by_container_properties(new_rules)
272
+ path_by_view = self._inheritance_path_by_view(new_data_model)
273
+ view_by_container_property = self._view_by_container_properties(new_data_model)
257
274
 
258
275
  parent_view_property_by_container_property = self._get_parent_view_property_by_container_property(
259
276
  path_by_view, view_by_container_property
260
277
  )
261
278
 
262
- for prop in new_rules.properties:
279
+ for prop in new_data_model.properties:
263
280
  if prop.container and prop.container_property:
264
281
  if parent_name := parent_view_property_by_container_property.get(
265
282
  (prop.container, prop.container_property)
266
283
  ):
267
284
  prop.view_property = parent_name
268
285
 
269
- return new_rules
286
+ return new_data_model
270
287
 
271
288
  # Todo: Move into Probe class. Note this means that the Probe class must take a NeatClient as an argument.
272
- def _inheritance_path_by_view(self, rules: DMSRules) -> dict[ViewEntity, list[ViewEntity]]:
273
- parents_by_view: dict[ViewEntity, list[ViewEntity]] = {view.view: view.implements or [] for view in rules.views}
289
+ def _inheritance_path_by_view(self, data_model: PhysicalDataModel) -> dict[ViewEntity, list[ViewEntity]]:
290
+ parents_by_view: dict[ViewEntity, list[ViewEntity]] = {
291
+ view.view: view.implements or [] for view in data_model.views
292
+ }
274
293
 
275
294
  path_by_view: dict[ViewEntity, list[ViewEntity]] = {}
276
- for view in rules.views:
295
+ for view in data_model.views:
277
296
  path_by_view[view.view] = self._get_inheritance_path(
278
- view.view, parents_by_view, rules.metadata.as_data_model_id()
297
+ view.view, parents_by_view, data_model.metadata.as_data_model_id()
279
298
  )
280
299
  return path_by_view
281
300
 
@@ -312,13 +331,13 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
312
331
  return inheritance_path
313
332
 
314
333
  def _view_by_container_properties(
315
- self, rules: DMSRules
334
+ self, data_model: PhysicalDataModel
316
335
  ) -> dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]]:
317
336
  view_properties_by_container_properties: dict[tuple[ContainerEntity, str], list[tuple[ViewEntity, str]]] = (
318
337
  defaultdict(list)
319
338
  )
320
339
  view_with_properties: set[ViewEntity] = set()
321
- for prop in rules.properties:
340
+ for prop in data_model.properties:
322
341
  if not prop.container or not prop.container_property:
323
342
  continue
324
343
  view_properties_by_container_properties[(prop.container, prop.container_property)].append(
@@ -327,7 +346,7 @@ class AsParentPropertyId(VerifiedRulesTransformer[DMSRules, DMSRules]):
327
346
  view_with_properties.add(prop.view)
328
347
 
329
348
  # We need to look up all parent properties.
330
- to_lookup = {view.view.as_id() for view in rules.views if view.view not in view_with_properties}
349
+ to_lookup = {view.view.as_id() for view in data_model.views if view.view not in view_with_properties}
331
350
  if to_lookup and self._client is None:
332
351
  raise CDFMissingClientError(
333
352
  f"Views {to_lookup} are not in the data model. Please provide a client to lookup the views."
@@ -0,0 +1,120 @@
1
+ from abc import ABC
2
+ from typing import cast
3
+
4
+ from cognite.neat.core._client import NeatClient
5
+ from cognite.neat.core._data_model._shared import (
6
+ ImportedDataModel,
7
+ T_ImportedUnverifiedDataModel,
8
+ T_VerifiedDataModel,
9
+ VerifiedDataModel,
10
+ )
11
+ from cognite.neat.core._data_model.models import (
12
+ ConceptualDataModel,
13
+ PhysicalDataModel,
14
+ UnverifiedConceptualDataModel,
15
+ UnverifiedPhysicalDataModel,
16
+ )
17
+ from cognite.neat.core._data_model.models.conceptual import ConceptualValidation
18
+ from cognite.neat.core._data_model.models.physical import PhysicalValidation
19
+ from cognite.neat.core._issues import MultiValueError, catch_issues
20
+ from cognite.neat.core._issues.errors import NeatTypeError, NeatValueError
21
+
22
+ from ._base import DataModelTransformer
23
+
24
+
25
+ class VerificationTransformer(DataModelTransformer[T_ImportedUnverifiedDataModel, T_VerifiedDataModel], ABC):
26
+ """Base class for all verification transformers."""
27
+
28
+ _data_model_cls: type[T_VerifiedDataModel]
29
+ _validation_cls: type
30
+
31
+ def __init__(self, validate: bool = True, client: NeatClient | None = None) -> None:
32
+ self.validate = validate
33
+ self._client = client
34
+
35
+ def transform(self, data_model: T_ImportedUnverifiedDataModel) -> T_VerifiedDataModel:
36
+ in_ = data_model.unverified_data_model
37
+ if in_ is None:
38
+ raise NeatValueError("Cannot verify rules. The reading of the rules failed.")
39
+ verified_data_model: T_VerifiedDataModel | None = None
40
+ # We need to catch issues as we use the error args to provide extra context for the errors/warnings
41
+ # For example, which row in the spreadsheet the error occurred.
42
+ with catch_issues(data_model.context) as issues:
43
+ data_model_cls = self._get_data_model_cls(data_model)
44
+ dumped = in_.dump()
45
+ verified_data_model = data_model_cls.model_validate(dumped) # type: ignore[assignment]
46
+ if self.validate:
47
+ validation_cls = self._get_validation_cls(verified_data_model) # type: ignore[arg-type]
48
+ if issubclass(validation_cls, PhysicalValidation):
49
+ validation_issues = PhysicalValidation(
50
+ cast(PhysicalDataModel, verified_data_model),
51
+ self._client,
52
+ data_model.context,
53
+ ).validate() # type: ignore[arg-type]
54
+ elif issubclass(validation_cls, ConceptualValidation):
55
+ validation_issues = ConceptualValidation(verified_data_model, data_model.context).validate() # type: ignore[arg-type]
56
+ else:
57
+ raise NeatValueError("Unsupported rule type")
58
+ issues.extend(validation_issues)
59
+
60
+ # Raise issues which is expected to be handled outside of this method
61
+ issues.trigger_warnings()
62
+ if issues.has_errors:
63
+ raise MultiValueError(issues.errors)
64
+ if verified_data_model is None:
65
+ raise NeatValueError("Data model was not verified")
66
+ return verified_data_model
67
+
68
+ def _get_data_model_cls(self, in_: T_ImportedUnverifiedDataModel) -> type[T_VerifiedDataModel]:
69
+ return self._data_model_cls
70
+
71
+ def _get_validation_cls(self, data_model: T_VerifiedDataModel) -> type:
72
+ return self._validation_cls
73
+
74
+ @property
75
+ def description(self) -> str:
76
+ return "Verify data model"
77
+
78
+
79
+ class VerifyPhysicalDataModel(
80
+ VerificationTransformer[ImportedDataModel[UnverifiedPhysicalDataModel], PhysicalDataModel]
81
+ ):
82
+ """Class to verify physical data model."""
83
+
84
+ _data_model_cls = PhysicalDataModel
85
+ _validation_cls = PhysicalValidation
86
+
87
+ def transform(self, data_model: ImportedDataModel[UnverifiedPhysicalDataModel]) -> PhysicalDataModel:
88
+ return super().transform(data_model)
89
+
90
+
91
+ class VerifyConceptualDataModel(
92
+ VerificationTransformer[ImportedDataModel[UnverifiedConceptualDataModel], ConceptualDataModel]
93
+ ):
94
+ """Class to verify conceptual data model."""
95
+
96
+ _data_model_cls = ConceptualDataModel
97
+ _validation_cls = ConceptualValidation
98
+
99
+ def transform(self, data_model: ImportedDataModel[UnverifiedConceptualDataModel]) -> ConceptualDataModel:
100
+ return super().transform(data_model)
101
+
102
+
103
+ class VerifyAnyDataModel(VerificationTransformer[T_ImportedUnverifiedDataModel, VerifiedDataModel]):
104
+ """Class to verify arbitrary data model"""
105
+
106
+ def _get_data_model_cls(self, in_: T_ImportedUnverifiedDataModel) -> type[VerifiedDataModel]:
107
+ if isinstance(in_.unverified_data_model, UnverifiedConceptualDataModel):
108
+ return ConceptualDataModel
109
+ elif isinstance(in_.unverified_data_model, UnverifiedPhysicalDataModel):
110
+ return PhysicalDataModel
111
+ else:
112
+ raise NeatTypeError(f"Unsupported data model type: {type(in_)}")
113
+
114
+ def _get_validation_cls(self, data_model: VerifiedDataModel) -> type:
115
+ if isinstance(data_model, ConceptualDataModel):
116
+ return ConceptualValidation
117
+ elif isinstance(data_model, PhysicalDataModel):
118
+ return PhysicalValidation
119
+ else:
120
+ raise NeatTypeError(f"Unsupported data model type: {type(data_model)}")
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
5
5
  from rdflib import URIRef
6
6
 
7
7
  from cognite.neat.core._constants import DEFAULT_NAMESPACE
8
- from cognite.neat.core._rules.models import InformationRules
8
+ from cognite.neat.core._data_model.models import ConceptualDataModel
9
9
  from cognite.neat.core._shared import Triple
10
10
  from cognite.neat.core._utils.auxiliary import class_html_doc
11
11
 
@@ -38,7 +38,7 @@ class KnowledgeGraphExtractor(BaseExtractor):
38
38
  """A knowledge graph extractor extracts triples with a schema"""
39
39
 
40
40
  @abstractmethod
41
- def get_information_rules(self) -> InformationRules:
41
+ def get_conceptual_data_model(self) -> ConceptualDataModel:
42
42
  """Returns the information rules that the extractor uses."""
43
43
  raise NotImplementedError()
44
44
 
@@ -18,7 +18,7 @@ from rdflib import RDF, XSD, Literal, Namespace, URIRef
18
18
  from typing_extensions import Self
19
19
 
20
20
  from cognite.neat.core._constants import DEFAULT_NAMESPACE
21
- from cognite.neat.core._graph.extractors._base import BaseExtractor
21
+ from cognite.neat.core._instances.extractors._base import BaseExtractor
22
22
  from cognite.neat.core._issues.errors import NeatValueError
23
23
  from cognite.neat.core._issues.warnings import CDFAuthWarning, NeatValueWarning
24
24
  from cognite.neat.core._shared import Triple
@@ -14,12 +14,15 @@ from cognite.neat.core._constants import (
14
14
  DEFAULT_NAMESPACE,
15
15
  get_default_prefixes_and_namespaces,
16
16
  )
17
- from cognite.neat.core._graph.extractors._base import KnowledgeGraphExtractor
17
+ from cognite.neat.core._data_model._shared import ImportedDataModel
18
+ from cognite.neat.core._data_model.catalog import classic_model
19
+ from cognite.neat.core._data_model.models import (
20
+ ConceptualDataModel,
21
+ UnverifiedConceptualDataModel,
22
+ )
23
+ from cognite.neat.core._instances.extractors._base import KnowledgeGraphExtractor
18
24
  from cognite.neat.core._issues.errors import NeatValueError, ResourceNotFoundError
19
25
  from cognite.neat.core._issues.warnings import CDFAuthWarning, NeatValueWarning
20
- from cognite.neat.core._rules._shared import ReadRules
21
- from cognite.neat.core._rules.catalog import classic_model
22
- from cognite.neat.core._rules.models import InformationInputRules, InformationRules
23
26
  from cognite.neat.core._shared import Triple
24
27
  from cognite.neat.core._utils.collection_ import chunker, iterate_progress_bar
25
28
  from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri
@@ -205,15 +208,18 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
205
208
 
206
209
  yield from self._extract_asset_parent_data_sets()
207
210
 
208
- def get_information_rules(self) -> InformationRules:
211
+ def get_conceptual_data_model(self) -> ConceptualDataModel:
209
212
  # To avoid circular imports
210
- from cognite.neat.core._rules.importers import ExcelImporter
213
+ from cognite.neat.core._data_model.importers import ExcelImporter
211
214
 
212
- unverified = cast(ReadRules[InformationInputRules], ExcelImporter(classic_model).to_rules())
213
- if unverified.rules is None:
215
+ unverified = cast(
216
+ ImportedDataModel[UnverifiedConceptualDataModel],
217
+ ExcelImporter(classic_model).to_data_model(),
218
+ )
219
+ if unverified.unverified_data_model is None:
214
220
  raise NeatValueError(f"Could not read the classic model rules from {classic_model}.")
215
221
 
216
- verified = unverified.rules.as_verified_rules()
222
+ verified = unverified.unverified_data_model.as_verified_data_model()
217
223
  prefixes = get_default_prefixes_and_namespaces()
218
224
  instance_prefix: str | None = next((k for k, v in prefixes.items() if v == self._namespace), None)
219
225
  if instance_prefix is None:
@@ -228,8 +234,8 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
228
234
  if is_snake_case:
229
235
  prop_id = to_snake_case(prop_id)
230
236
  prop.instance_source = [self._namespace[prop_id]]
231
- for cls_ in verified.classes:
232
- cls_id = cls_.class_.suffix
237
+ for cls_ in verified.concepts:
238
+ cls_id = cls_.concept.suffix
233
239
  if is_snake_case:
234
240
  cls_id = to_snake_case(cls_id)
235
241
  cls_.instance_source = self._namespace[cls_id]
@@ -7,18 +7,21 @@ from rdflib import Namespace, URIRef
7
7
 
8
8
  from cognite.neat.core._client import NeatClient
9
9
  from cognite.neat.core._constants import COGNITE_SPACES, DEFAULT_NAMESPACE
10
+ from cognite.neat.core._data_model.importers import DMSImporter
11
+ from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
12
+ from cognite.neat.core._data_model.models.conceptual import ConceptualProperty
13
+ from cognite.neat.core._data_model.models.data_types import Json
14
+ from cognite.neat.core._data_model.models.entities import UnknownEntity
15
+ from cognite.neat.core._data_model.transformers import (
16
+ PhysicalToConceptual,
17
+ VerifyPhysicalDataModel,
18
+ )
10
19
  from cognite.neat.core._issues import IssueList, NeatIssue, catch_warnings
11
20
  from cognite.neat.core._issues.warnings import (
12
21
  CDFAuthWarning,
13
22
  ResourceNotFoundWarning,
14
23
  ResourceRetrievalWarning,
15
24
  )
16
- from cognite.neat.core._rules.importers import DMSImporter
17
- from cognite.neat.core._rules.models import DMSRules, InformationRules
18
- from cognite.neat.core._rules.models.data_types import Json
19
- from cognite.neat.core._rules.models.entities import UnknownEntity
20
- from cognite.neat.core._rules.models.information import InformationProperty
21
- from cognite.neat.core._rules.transformers import DMSToInformation, VerifyDMSRules
22
25
  from cognite.neat.core._shared import Triple
23
26
 
24
27
  from ._base import KnowledgeGraphExtractor
@@ -47,8 +50,8 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
47
50
  self._str_to_ideal_type = str_to_ideal_type
48
51
 
49
52
  self._views: list[dm.View] | None = None
50
- self._information_rules: InformationRules | None = None
51
- self._dms_rules: DMSRules | None = None
53
+ self._conceptual_data_model: ConceptualDataModel | None = None
54
+ self._physical_data_model: PhysicalDataModel | None = None
52
55
 
53
56
  @classmethod
54
57
  def from_data_model_id(
@@ -164,32 +167,32 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
164
167
  self._issues.append(ResourceNotFoundWarning(dm_view, "view", data_model_id, "data model"))
165
168
  return views
166
169
 
167
- def get_information_rules(self) -> InformationRules:
168
- """Returns the information rules that the extractor uses."""
169
- if self._information_rules is None:
170
- self._information_rules, self._dms_rules = self._create_rules()
171
- return self._information_rules
170
+ def get_conceptual_data_model(self) -> ConceptualDataModel:
171
+ """Returns the conceptual data model that the extractor uses."""
172
+ if self._conceptual_data_model is None:
173
+ self._conceptual_data_model, self._physical_data_model = self._create_data_models()
174
+ return self._conceptual_data_model
172
175
 
173
- def get_dms_rules(self) -> DMSRules:
174
- """Returns the DMS rules that the extractor uses."""
175
- if self._dms_rules is None:
176
- self._information_rules, self._dms_rules = self._create_rules()
177
- return self._dms_rules
176
+ def get_physical_data_model(self) -> PhysicalDataModel:
177
+ """Returns the physical data model that the extractor uses."""
178
+ if self._physical_data_model is None:
179
+ self._conceptual_data_model, self._physical_data_model = self._create_data_models()
180
+ return self._physical_data_model
178
181
 
179
182
  def get_issues(self) -> IssueList:
180
183
  """Returns the issues that occurred during the extraction."""
181
184
  return self._issues
182
185
 
183
- def _create_rules(self) -> tuple[InformationRules, DMSRules]:
184
- # The DMS and Information rules must be created together to link them property.
186
+ def _create_data_models(self) -> tuple[ConceptualDataModel, PhysicalDataModel]:
187
+ # The physical and conceptual data model must be created together to link them property.
185
188
  importer = DMSImporter.from_data_model(self._client, self._data_model)
186
- unverified_dms = importer.to_rules()
187
- if self._unpack_json and (dms_rules := unverified_dms.rules):
188
- # Drop the JSON properties from the DMS rules as these are no longer valid.
189
+ imported_physical_data_model = importer.to_data_model()
190
+ if self._unpack_json and (unverified_physical_data_model := imported_physical_data_model.unverified_data_model):
191
+ # Drop the JSON properties from the physical data model as these are no longer valid.
189
192
  json_name = Json().name # To avoid instantiating Json multiple times.
190
- dms_rules.properties = [
193
+ unverified_physical_data_model.properties = [
191
194
  prop
192
- for prop in dms_rules.properties
195
+ for prop in unverified_physical_data_model.properties
193
196
  if not (
194
197
  (
195
198
  isinstance(prop.value_type, Json)
@@ -202,23 +205,28 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
202
205
 
203
206
  with catch_warnings() as issues:
204
207
  # Any errors occur will be raised and caught outside the extractor.
205
- verified_dms = VerifyDMSRules(client=self._client).transform(unverified_dms)
206
- information_rules = DMSToInformation(self._namespace).transform(verified_dms)
208
+ verified_physical_data_model = VerifyPhysicalDataModel(client=self._client).transform(
209
+ imported_physical_data_model
210
+ )
211
+ verified_conceptual_data_model = PhysicalToConceptual(self._namespace).transform(
212
+ verified_physical_data_model
213
+ )
207
214
 
208
- # We need to sync the metadata between the two rules, such that the `.sync_with_info_rules` method works.
209
- information_rules.metadata.physical = verified_dms.metadata.identifier
210
- verified_dms.metadata.logical = information_rules.metadata.identifier
211
- verified_dms.sync_with_info_rules(information_rules)
215
+ # We need to sync the metadata between the two data model, such that the
216
+ # `.sync_with_conceptual_data_model` method works.
217
+ verified_conceptual_data_model.metadata.physical = verified_physical_data_model.metadata.identifier
218
+ verified_physical_data_model.metadata.conceptual = verified_conceptual_data_model.metadata.identifier
219
+ verified_physical_data_model.sync_with_conceptual_data_model(verified_conceptual_data_model)
212
220
 
213
- # Adding startNode and endNode to the information rules for views that are used for edges.
214
- classes_by_prefix = {cls_.class_.prefix: cls_ for cls_ in information_rules.classes}
221
+ # Adding startNode and endNode to the conceptual data model for views that are used for edges.
222
+ concepts_by_prefix = {concept.concept.prefix: concept for concept in verified_conceptual_data_model.concepts}
215
223
  for view in self._model_views:
216
- if view.used_for == "edge" and view.external_id in classes_by_prefix:
217
- cls_ = classes_by_prefix[view.external_id]
224
+ if view.used_for == "edge" and view.external_id in concepts_by_prefix:
225
+ cls_ = concepts_by_prefix[view.external_id]
218
226
  for property_ in ("startNode", "endNode"):
219
- information_rules.properties.append(
220
- InformationProperty(
221
- class_=cls_.class_,
227
+ verified_conceptual_data_model.properties.append(
228
+ ConceptualProperty(
229
+ concept=cls_.concept,
222
230
  property_=property_,
223
231
  value_type=UnknownEntity(),
224
232
  min_count=0,
@@ -227,4 +235,4 @@ class DMSGraphExtractor(KnowledgeGraphExtractor):
227
235
  )
228
236
 
229
237
  self._issues.extend(issues)
230
- return information_rules, verified_dms
238
+ return verified_conceptual_data_model, verified_physical_data_model