cognite-neat 0.104.0__py3-none-any.whl → 0.105.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (141) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
  2. cognite/neat/_client/_api/schema.py +2 -1
  3. cognite/neat/_client/data_classes/neat_sequence.py +261 -0
  4. cognite/neat/_client/data_classes/schema.py +5 -1
  5. cognite/neat/_client/testing.py +33 -0
  6. cognite/neat/_constants.py +56 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
  8. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
  9. cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
  10. cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
  11. cognite/neat/_graph/transformers/__init__.py +3 -1
  12. cognite/neat/_graph/transformers/_classic_cdf.py +2 -1
  13. cognite/neat/_graph/transformers/_value_type.py +72 -0
  14. cognite/neat/_issues/__init__.py +0 -2
  15. cognite/neat/_issues/_base.py +19 -35
  16. cognite/neat/_issues/warnings/__init__.py +4 -1
  17. cognite/neat/_issues/warnings/_general.py +7 -0
  18. cognite/neat/_issues/warnings/_resources.py +11 -0
  19. cognite/neat/_rules/exporters/_rules2dms.py +35 -1
  20. cognite/neat/_rules/exporters/_rules2excel.py +2 -2
  21. cognite/neat/_rules/importers/_dms2rules.py +66 -55
  22. cognite/neat/_rules/models/_base_rules.py +4 -1
  23. cognite/neat/_rules/models/entities/_wrapped.py +10 -5
  24. cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
  25. cognite/neat/_rules/transformers/__init__.py +8 -2
  26. cognite/neat/_rules/transformers/_converters.py +271 -188
  27. cognite/neat/_rules/transformers/_mapping.py +75 -59
  28. cognite/neat/_rules/transformers/_verification.py +2 -3
  29. cognite/neat/_session/_inspect.py +3 -1
  30. cognite/neat/_session/_prepare.py +112 -24
  31. cognite/neat/_session/_read.py +33 -70
  32. cognite/neat/_session/_state.py +2 -2
  33. cognite/neat/_session/_to.py +2 -2
  34. cognite/neat/_store/_rules_store.py +4 -8
  35. cognite/neat/_utils/reader/_base.py +27 -0
  36. cognite/neat/_version.py +1 -1
  37. {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
  38. cognite_neat-0.105.0.dist-info/RECORD +179 -0
  39. {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
  40. cognite/neat/_app/api/__init__.py +0 -0
  41. cognite/neat/_app/api/asgi/metrics.py +0 -4
  42. cognite/neat/_app/api/configuration.py +0 -98
  43. cognite/neat/_app/api/context_manager/__init__.py +0 -3
  44. cognite/neat/_app/api/context_manager/manager.py +0 -16
  45. cognite/neat/_app/api/data_classes/__init__.py +0 -0
  46. cognite/neat/_app/api/data_classes/rest.py +0 -59
  47. cognite/neat/_app/api/explorer.py +0 -66
  48. cognite/neat/_app/api/routers/configuration.py +0 -25
  49. cognite/neat/_app/api/routers/crud.py +0 -102
  50. cognite/neat/_app/api/routers/metrics.py +0 -10
  51. cognite/neat/_app/api/routers/workflows.py +0 -224
  52. cognite/neat/_app/api/utils/__init__.py +0 -0
  53. cognite/neat/_app/api/utils/data_mapping.py +0 -17
  54. cognite/neat/_app/api/utils/logging.py +0 -26
  55. cognite/neat/_app/api/utils/query_templates.py +0 -92
  56. cognite/neat/_app/main.py +0 -17
  57. cognite/neat/_app/monitoring/__init__.py +0 -0
  58. cognite/neat/_app/monitoring/metrics.py +0 -69
  59. cognite/neat/_app/ui/index.html +0 -1
  60. cognite/neat/_app/ui/neat-app/.gitignore +0 -23
  61. cognite/neat/_app/ui/neat-app/README.md +0 -70
  62. cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
  63. cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
  64. cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
  65. cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
  66. cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
  67. cognite/neat/_app/ui/neat-app/build/index.html +0 -1
  68. cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
  69. cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
  70. cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
  71. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
  72. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
  73. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
  74. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
  75. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
  76. cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
  77. cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
  78. cognite/neat/_app/ui/neat-app/package.json +0 -62
  79. cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
  80. cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
  81. cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
  82. cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
  83. cognite/neat/_app/ui/neat-app/public/index.html +0 -43
  84. cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
  85. cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
  86. cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
  87. cognite/neat/_app/ui/neat-app/src/App.css +0 -38
  88. cognite/neat/_app/ui/neat-app/src/App.js +0 -17
  89. cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
  90. cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
  91. cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
  92. cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
  93. cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
  94. cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
  95. cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
  96. cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
  97. cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
  98. cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
  99. cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
  100. cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
  101. cognite/neat/_app/ui/neat-app/src/index.css +0 -13
  102. cognite/neat/_app/ui/neat-app/src/index.js +0 -13
  103. cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
  104. cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
  105. cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
  106. cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
  107. cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
  108. cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
  109. cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
  110. cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
  111. cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
  112. cognite/neat/_workflows/__init__.py +0 -17
  113. cognite/neat/_workflows/base.py +0 -590
  114. cognite/neat/_workflows/cdf_store.py +0 -393
  115. cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
  116. cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
  117. cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
  118. cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
  119. cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  120. cognite/neat/_workflows/manager.py +0 -292
  121. cognite/neat/_workflows/model.py +0 -203
  122. cognite/neat/_workflows/steps/__init__.py +0 -0
  123. cognite/neat/_workflows/steps/data_contracts.py +0 -109
  124. cognite/neat/_workflows/steps/lib/__init__.py +0 -0
  125. cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
  126. cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
  127. cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
  128. cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
  129. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
  130. cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -323
  131. cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
  132. cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
  133. cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
  134. cognite/neat/_workflows/steps/step_model.py +0 -79
  135. cognite/neat/_workflows/steps_registry.py +0 -218
  136. cognite/neat/_workflows/tasks.py +0 -18
  137. cognite/neat/_workflows/triggers.py +0 -169
  138. cognite/neat/_workflows/utils.py +0 -19
  139. cognite_neat-0.104.0.dist-info/RECORD +0 -276
  140. {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
  141. {cognite_neat-0.104.0.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
@@ -1,18 +1,17 @@
1
1
  import warnings
2
2
  from abc import ABC
3
3
  from collections import defaultdict
4
- from functools import cached_property
5
4
  from typing import Any, ClassVar, Literal
6
5
 
7
6
  from cognite.client import data_modeling as dm
8
7
 
9
8
  from cognite.neat._client import NeatClient
10
9
  from cognite.neat._issues.errors import CDFMissingClientError, NeatValueError, ResourceNotFoundError
11
- from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
10
+ from cognite.neat._issues.warnings import PropertyOverwritingWarning
12
11
  from cognite.neat._rules.models import DMSRules, SheetList
13
12
  from cognite.neat._rules.models.data_types import Enum
14
- from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
15
- from cognite.neat._rules.models.entities import ContainerEntity, ViewEntity
13
+ from cognite.neat._rules.models.dms import DMSContainer, DMSEnum, DMSProperty
14
+ from cognite.neat._rules.models.entities import ClassEntity, ContainerEntity, ViewEntity
16
15
 
17
16
  from ._base import RulesTransformer
18
17
 
@@ -104,11 +103,12 @@ class MapOneToOne(MapOntoTransformers):
104
103
  class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
105
104
  """Maps properties and classes using the given mapping.
106
105
 
107
- **Note**: This transformer mutates the input rules.
108
-
109
106
  Args:
110
- mapping: The mapping to use.
111
-
107
+ mapping: The mapping to use represented as a DMSRules object.
108
+ data_type_conflict: How to handle data type conflicts. The default is "overwrite".
109
+ A data type conflicts occurs when the data type of a property in the mapping is different from the
110
+ data type of the property in the input rules. If "overwrite" the data type in the input rules is overwritten
111
+ with the data type in the mapping.
112
112
  """
113
113
 
114
114
  _mapping_fields: ClassVar[frozenset[str]] = frozenset(
@@ -119,67 +119,83 @@ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
119
119
  self.mapping = mapping
120
120
  self.data_type_conflict = data_type_conflict
121
121
 
122
- @cached_property
123
- def _view_by_entity_id(self) -> dict[str, DMSView]:
124
- return {view.view.external_id: view for view in self.mapping.views}
125
-
126
- @cached_property
127
- def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
128
- return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
129
-
130
122
  def transform(self, rules: DMSRules) -> DMSRules:
131
123
  if self.data_type_conflict != "overwrite":
132
124
  raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
133
125
  input_rules = rules
134
126
  new_rules = input_rules.model_copy(deep=True)
135
127
 
136
- for view in new_rules.views:
137
- if mapping_view := self._view_by_entity_id.get(view.view.external_id):
138
- view.implements = mapping_view.implements
128
+ views_by_external_id = {view.view.external_id: view for view in new_rules.views}
129
+ new_views: set[ViewEntity] = set()
130
+ for mapping_view in self.mapping.views:
131
+ if existing_view := views_by_external_id.get(mapping_view.view.external_id):
132
+ existing_view.implements = mapping_view.implements
133
+ else:
134
+ # We need to add all the views in the mapping that are not in the input rules.
135
+ # This is to ensure that all ValueTypes are present in the resulting rules.
136
+ # For example, if a property is a direct relation to an Equipment view, we need to add
137
+ # the Equipment view to the rules.
138
+ new_rules.views.append(mapping_view)
139
+ new_views.add(mapping_view.view)
139
140
 
140
- for prop in new_rules.properties:
141
- key = (prop.view.external_id, prop.view_property)
142
- if key not in self._property_by_view_property:
143
- continue
144
- mapping_prop = self._property_by_view_property[key]
145
- to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
146
- if conflicts and self.data_type_conflict == "overwrite":
147
- warnings.warn(
148
- PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
149
- stacklevel=2,
150
- )
151
- elif conflicts:
152
- raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
153
- for field_name, value in to_overwrite.items():
154
- setattr(prop, field_name, value)
155
- prop.container = mapping_prop.container
156
- prop.container_property = mapping_prop.container_property
157
-
158
- # Add missing views used as value types
159
- existing_views = {view.view for view in new_rules.views}
160
- new_value_types = {
161
- prop.value_type
162
- for prop in new_rules.properties
163
- if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
141
+ properties_by_view_property = {
142
+ (prop.view.external_id, prop.view_property): prop for prop in new_rules.properties
164
143
  }
165
- for new_value_type in new_value_types:
166
- if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
167
- new_rules.views.append(mapping_view)
144
+ existing_enum_collections = {item.collection for item in new_rules.enum or []}
145
+ mapping_enums_by_collection: dict[ClassEntity, list[DMSEnum]] = defaultdict(list)
146
+ for item in self.mapping.enum or []:
147
+ mapping_enums_by_collection[item.collection].append(item)
148
+ existing_containers = {container.container for container in new_rules.containers or []}
149
+ mapping_containers_by_id = {container.container: container for container in self.mapping.containers or []}
150
+ for mapping_prop in self.mapping.properties:
151
+ if existing_prop := properties_by_view_property.get(
152
+ (mapping_prop.view.external_id, mapping_prop.view_property)
153
+ ):
154
+ to_overwrite, conflicts = self._find_overwrites(existing_prop, mapping_prop)
155
+ if conflicts and self.data_type_conflict == "overwrite":
156
+ warnings.warn(
157
+ PropertyOverwritingWarning(
158
+ existing_prop.view.as_id(), "view", existing_prop.view_property, tuple(conflicts)
159
+ ),
160
+ stacklevel=2,
161
+ )
162
+ elif conflicts:
163
+ raise NeatValueError(
164
+ f"Conflicting properties for {existing_prop.view}.{existing_prop.view_property}: {conflicts}"
165
+ )
166
+
167
+ for field_name, value in to_overwrite.items():
168
+ setattr(existing_prop, field_name, value)
169
+ existing_prop.container = mapping_prop.container
170
+ existing_prop.container_property = mapping_prop.container_property
171
+ elif isinstance(mapping_prop.value_type, ViewEntity):
172
+ # All connections must be included in the rules. This is to update the
173
+ # ValueTypes of the implemented views.
174
+ new_rules.properties.append(mapping_prop)
175
+ elif mapping_prop.view in new_views:
176
+ # All properties of new views are included. Main motivation is GUIDs properties
177
+ new_rules.properties.append(mapping_prop)
168
178
  else:
169
- warnings.warn(NeatValueWarning(f"View {new_value_type} not found in mapping"), stacklevel=2)
179
+ # Skipping mapped properties that are not in the input rules.
180
+ continue
170
181
 
171
- # Add missing enums
172
- existing_enum_collections = {item.collection for item in new_rules.enum or []}
173
- new_enums = {
174
- prop.value_type.collection
175
- for prop in new_rules.properties
176
- if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
177
- }
178
- if new_enums:
179
- new_rules.enum = new_rules.enum or SheetList[DMSEnum]([])
180
- for item in self.mapping.enum or []:
181
- if item.collection in new_enums:
182
- new_rules.enum.append(item)
182
+ if (
183
+ isinstance(mapping_prop.value_type, Enum)
184
+ and mapping_prop.value_type.collection not in existing_enum_collections
185
+ ):
186
+ if not new_rules.enum:
187
+ new_rules.enum = SheetList[DMSEnum]([])
188
+ new_rules.enum.extend(mapping_enums_by_collection[mapping_prop.value_type.collection])
189
+
190
+ if (
191
+ mapping_prop.container
192
+ and mapping_prop.container not in existing_containers
193
+ and (new_container := mapping_containers_by_id.get(mapping_prop.container))
194
+ ):
195
+ # Mapping can include new containers for GUID properties
196
+ if not new_rules.containers:
197
+ new_rules.containers = SheetList[DMSContainer]([])
198
+ new_rules.containers.append(new_container)
183
199
 
184
200
  return new_rules
185
201
 
@@ -1,7 +1,7 @@
1
1
  from abc import ABC
2
2
 
3
3
  from cognite.neat._client import NeatClient
4
- from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning, catch_issues
4
+ from cognite.neat._issues import MultiValueError, catch_issues
5
5
  from cognite.neat._issues.errors import NeatTypeError, NeatValueError
6
6
  from cognite.neat._rules._shared import (
7
7
  ReadRules,
@@ -39,8 +39,7 @@ class VerificationTransformer(RulesTransformer[T_ReadInputRules, T_VerifiedRules
39
39
  verified_rules: T_VerifiedRules | None = None
40
40
  # We need to catch issues as we use the error args to provide extra context for the errors/warnings
41
41
  # For example, which row in the spreadsheet the error occurred o
42
- issues = IssueList()
43
- with catch_issues(issues, NeatError, NeatWarning, error_args) as _:
42
+ with catch_issues(error_args=error_args) as issues:
44
43
  rules_cls = self._get_rules_cls(rules)
45
44
  dumped = in_.dump()
46
45
  verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
@@ -59,7 +59,9 @@ class InspectAPI:
59
59
  neat.inspect.properties
60
60
  ```
61
61
  """
62
- return self._state.rule_store.last_verified_rule.properties.to_pandas()
62
+ df = self._state.rule_store.last_verified_rule.properties.to_pandas()
63
+ df.drop(columns=["neatId"], errors="ignore", inplace=True)
64
+ return df
63
65
 
64
66
 
65
67
  @session_class_wrapper
@@ -8,8 +8,10 @@ from cognite.neat._client import NeatClient
8
8
  from cognite.neat._constants import (
9
9
  get_default_prefixes_and_namespaces,
10
10
  )
11
+ from cognite.neat._graph import extractors
11
12
  from cognite.neat._graph.transformers import (
12
13
  AttachPropertyFromTargetToSource,
14
+ ConnectionToLiteral,
13
15
  ConvertLiteral,
14
16
  LiteralToEntity,
15
17
  PruneDeadEndEdges,
@@ -20,14 +22,20 @@ from cognite.neat._graph.transformers import (
20
22
  )
21
23
  from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
22
24
  from cognite.neat._issues import IssueList
25
+ from cognite.neat._issues.errors import NeatValueError
26
+ from cognite.neat._rules.models.dms import DMSValidation
23
27
  from cognite.neat._rules.transformers import (
24
28
  AddClassImplements,
25
29
  IncludeReferenced,
26
30
  PrefixEntities,
27
31
  ReduceCogniteModel,
32
+ RulesTransformer,
28
33
  ToCompliantEntities,
29
- ToExtension,
34
+ ToDataProductModel,
35
+ ToEnterpriseModel,
36
+ ToSolutionModel,
30
37
  )
38
+ from cognite.neat._utils.text import humanize_collection
31
39
 
32
40
  from ._state import SessionState
33
41
  from .exceptions import NeatSessionError, session_class_wrapper
@@ -195,9 +203,11 @@ class InstancePrepareAPI:
195
203
  neat.prepare.instances.make_connection_on_exact_match(source, target, connection)
196
204
  ```
197
205
  """
198
-
199
- subject_type, subject_predicate = self._get_type_and_property_uris(*source)
200
- object_type, object_predicate = self._get_type_and_property_uris(*target)
206
+ try:
207
+ subject_type, subject_predicate = self._get_type_and_property_uris(*source)
208
+ object_type, object_predicate = self._get_type_and_property_uris(*target)
209
+ except NeatValueError as e:
210
+ raise NeatSessionError(f"Cannot make connection: {e}") from None
201
211
 
202
212
  transformer = MakeConnectionOnExactMatch(
203
213
  subject_type,
@@ -215,17 +225,19 @@ class InstancePrepareAPI:
215
225
  property_uri = self._state.instances.store.queries.property_uri(property_)
216
226
 
217
227
  if not type_uri:
218
- raise NeatSessionError(f"Type {type_} does not exist in the graph.")
228
+ raise NeatValueError(f"Type {type_} does not exist in the graph.")
219
229
  elif len(type_uri) > 1:
220
- raise NeatSessionError(f"{type_} has multiple ids found in the graph: {','.join(type_uri)}.")
230
+ raise NeatValueError(f"{type_} has multiple ids found in the graph: {humanize_collection(type_uri)}.")
221
231
 
222
232
  if not property_uri:
223
- raise NeatSessionError(f"Property {property_} does not exist in the graph.")
233
+ raise NeatValueError(f"Property {property_} does not exist in the graph.")
224
234
  elif len(type_uri) > 1:
225
- raise NeatSessionError(f"{property_} has multiple ids found in the graph: {','.join(property_uri)}.")
235
+ raise NeatValueError(
236
+ f"{property_} has multiple ids found in the graph: {humanize_collection(property_uri)}."
237
+ )
226
238
 
227
239
  if not self._state.instances.store.queries.type_with_property(type_uri[0], property_uri[0]):
228
- raise NeatSessionError(f"Property {property_} is not defined for type {type_}. Cannot make connection")
240
+ raise NeatValueError(f"Property {property_} is not defined for type {type_}.")
229
241
  return type_uri[0], property_uri[0]
230
242
 
231
243
  def relationships_as_edges(self, min_relationship_types: int = 1, limit_per_type: int | None = None) -> None:
@@ -269,7 +281,10 @@ class InstancePrepareAPI:
269
281
  ```
270
282
 
271
283
  """
272
- subject_type, subject_predicate = self._get_type_and_property_uris(*source)
284
+ try:
285
+ subject_type, subject_predicate = self._get_type_and_property_uris(*source)
286
+ except NeatValueError as e:
287
+ raise NeatSessionError(f"Cannot convert data type: {e}") from None
273
288
 
274
289
  transformer = ConvertLiteral(subject_type, subject_predicate, convert)
275
290
  self._state.instances.store.transform(transformer)
@@ -294,13 +309,81 @@ class InstancePrepareAPI:
294
309
  """
295
310
  subject_type: URIRef | None = None
296
311
  if source[0] is not None:
297
- subject_type, subject_predicate = self._get_type_and_property_uris(*source) # type: ignore[arg-type, assignment]
312
+ try:
313
+ subject_type, subject_predicate = self._get_type_and_property_uris(*source) # type: ignore[arg-type, assignment]
314
+ except NeatValueError as e:
315
+ raise NeatSessionError(f"Cannot convert to type: {e}") from None
298
316
  else:
299
317
  subject_predicate = self._state.instances.store.queries.property_uri(source[1])[0]
300
318
 
301
319
  transformer = LiteralToEntity(subject_type, subject_predicate, type, new_property)
302
320
  self._state.instances.store.transform(transformer)
303
321
 
322
+ def connection_to_data_type(self, source: tuple[str | None, str]) -> None:
323
+ """Converts a connection to a data type.
324
+
325
+ Args:
326
+ source: The source of the conversion. A tuple of (type, property)
327
+ where property is the property that should be converted.
328
+ You can pass (None, property) to covert all properties with the given name.
329
+
330
+ Example:
331
+
332
+ Convert all properties 'labels' from a connection to a string:
333
+
334
+ ```python
335
+ neat.prepare.instances.connection_to_data_type(
336
+ (None, "labels")
337
+ )
338
+ ```
339
+
340
+ """
341
+ subject_type: URIRef | None = None
342
+ if source[0] is not None:
343
+ try:
344
+ subject_type, subject_predicate = self._get_type_and_property_uris(*source) # type: ignore[arg-type, assignment]
345
+ except NeatValueError as e:
346
+ raise NeatSessionError(f"Cannot convert to data type: {e}") from None
347
+ else:
348
+ subject_predicate = self._state.instances.store.queries.property_uri(source[1])[0]
349
+ transformer = ConnectionToLiteral(subject_type, subject_predicate)
350
+ self._state.instances.store.transform(transformer)
351
+
352
+ def classic_to_core(self) -> None:
353
+ """Prepares extracted CDF classic graph for the Core Data model.
354
+
355
+ !!! note "This method bundles several graph transformers which"
356
+ - Convert relationships to edges
357
+ - Convert TimeSeries.type from bool to enum
358
+ - Convert all properties 'source' to a connection to SourceSystem
359
+ - Convert all properties 'labels' from a connection to a string
360
+
361
+ Example:
362
+ Apply classic to core transformations:
363
+ ```python
364
+ neat.prepare.instances.classic_to_core()
365
+ ```
366
+ """
367
+ self.relationships_as_edges()
368
+ self.convert_data_type(
369
+ ("TimeSeries", "isString"), convert=lambda is_string: "string" if is_string else "numeric"
370
+ )
371
+ self.property_to_type((None, "source"), "SourceSystem", "name")
372
+ for type_ in [
373
+ extractors.EventsExtractor._default_rdf_type,
374
+ extractors.AssetsExtractor._default_rdf_type,
375
+ extractors.FilesExtractor._default_rdf_type,
376
+ ]:
377
+ try:
378
+ subject_type, subject_predicate = self._get_type_and_property_uris(type_, "labels")
379
+ except NeatValueError:
380
+ # If the type_.labels does not exist, continue. This is not an error, it just means that the
381
+ # Labels is not used in the graph for that type.
382
+ continue
383
+ else:
384
+ transformer = ConnectionToLiteral(subject_type, subject_predicate)
385
+ self._state.instances.store.transform(transformer)
386
+
304
387
 
305
388
  @session_class_wrapper
306
389
  class DataModelPrepareAPI:
@@ -357,10 +440,9 @@ class DataModelPrepareAPI:
357
440
 
358
441
  """
359
442
  return self._state.rule_transform(
360
- ToExtension(
443
+ ToEnterpriseModel(
361
444
  new_model_id=data_model_id,
362
445
  org_name=org_name,
363
- type_="enterprise",
364
446
  dummy_property=dummy_property,
365
447
  move_connections=move_connections,
366
448
  )
@@ -394,10 +476,9 @@ class DataModelPrepareAPI:
394
476
 
395
477
  """
396
478
  return self._state.rule_transform(
397
- ToExtension(
479
+ ToSolutionModel(
398
480
  new_model_id=data_model_id,
399
481
  org_name=org_name,
400
- type_="solution",
401
482
  mode=mode,
402
483
  dummy_property=dummy_property,
403
484
  )
@@ -416,25 +497,32 @@ class DataModelPrepareAPI:
416
497
 
417
498
  Args:
418
499
  data_model_id: The data product data model id that is being created.
419
- org_name: Organization name to use for the views in the new data model.
500
+ org_name: Organization name used as prefix if the model is building on top of a Cognite Data Model.
420
501
  include: The views to include in the data product data model. Can be either "same-space" or "all".
421
- If you set same-space, only the views in the same space as the data model will be included.
502
+ If you set same-space, only the properties of the views in the same space as the data model
503
+ will be included.
422
504
  """
423
- if self._client is None:
505
+
506
+ view_ids, container_ids = DMSValidation(
507
+ self._state.rule_store.last_verified_dms_rules
508
+ ).imported_views_and_containers_ids()
509
+ transformers: list[RulesTransformer] = []
510
+ if (view_ids or container_ids) and self._client is None:
424
511
  raise NeatSessionError(
425
512
  "No client provided. You are referencing unknown views and containers in your data model, "
426
513
  "NEAT needs a client to lookup the definitions. "
427
514
  "Please set the client in the session, NeatSession(client=client)."
428
515
  )
429
- transformers = [
430
- IncludeReferenced(self._client, include_properties=True),
431
- ToExtension(
516
+ elif (view_ids or container_ids) and self._client:
517
+ transformers.append(IncludeReferenced(self._client, include_properties=True))
518
+
519
+ transformers.append(
520
+ ToDataProductModel(
432
521
  new_model_id=data_model_id,
433
522
  org_name=org_name,
434
- type_="data_product",
435
523
  include=include,
436
- ),
437
- ]
524
+ )
525
+ )
438
526
 
439
527
  self._state.rule_transform(*transformers)
440
528
 
@@ -1,18 +1,16 @@
1
- import tempfile
2
- from pathlib import Path
3
1
  from typing import Any, Literal, cast
4
2
 
5
3
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
6
4
 
7
5
  from cognite.neat._client import NeatClient
8
- from cognite.neat._constants import COGNITE_SPACES
9
6
  from cognite.neat._graph import examples as instances_examples
10
7
  from cognite.neat._graph import extractors
11
8
  from cognite.neat._issues import IssueList
12
9
  from cognite.neat._issues.errors import NeatValueError
10
+ from cognite.neat._issues.warnings import MissingCogniteClientWarning
13
11
  from cognite.neat._rules import catalog, importers
14
12
  from cognite.neat._rules.importers import BaseImporter
15
- from cognite.neat._utils.reader import GitHubReader, HttpFileReader, NeatReader, PathReader
13
+ from cognite.neat._utils.reader import NeatReader
16
14
 
17
15
  from ._state import SessionState
18
16
  from ._wizard import NeatObjectType, RDFFileType, XMLFileType, object_wizard, rdf_dm_wizard, xml_format_wizard
@@ -167,9 +165,8 @@ class ExcelReadAPI(BaseReadAPI):
167
165
  io: file path to the Excel sheet
168
166
  """
169
167
  reader = NeatReader.create(io)
170
- if not isinstance(reader, PathReader):
171
- raise NeatValueError("Only file paths are supported for Excel files")
172
- return self._state.rule_import(importers.ExcelImporter(reader.path))
168
+ path = reader.materialize_path()
169
+ return self._state.rule_import(importers.ExcelImporter(path))
173
170
 
174
171
 
175
172
  @session_class_wrapper
@@ -185,46 +182,32 @@ class ExcelExampleAPI(BaseReadAPI):
185
182
 
186
183
  @session_class_wrapper
187
184
  class YamlReadAPI(BaseReadAPI):
188
- """Reads a yaml with either neat rules, or several toolkit yaml files to import Data Model(s) into NeatSession.
189
-
190
- Args:
191
- io: file path to the Yaml file in the case of "neat" yaml, or path to a zip folder or directory with several
192
- Yaml files in the case of "toolkit".
185
+ def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
186
+ """Reads a yaml with either neat rules, or several toolkit yaml files to import Data Model(s) into NeatSession.
193
187
 
194
- Example:
195
- ```python
196
- neat.read.yaml("path_to_toolkit_yamls")
197
- ```
198
- """
188
+ Args:
189
+ io: File path to the Yaml file in the case of "neat" yaml, or path to a zip folder or directory with several
190
+ Yaml files in the case of "toolkit".
191
+ format: The format of the yaml file(s). Can be either "neat" or "toolkit".
199
192
 
200
- def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
193
+ Example:
194
+ ```python
195
+ neat.read.yaml("path_to_toolkit_yamls")
196
+ ```
197
+ """
201
198
  reader = NeatReader.create(io)
202
- if not isinstance(reader, PathReader):
203
- raise NeatValueError("Only file paths are supported for YAML files")
199
+ path = reader.materialize_path()
204
200
  importer: BaseImporter
205
201
  if format == "neat":
206
- importer = importers.YAMLImporter.from_file(reader.path, source_name=f"{reader!s}")
202
+ importer = importers.YAMLImporter.from_file(path, source_name=f"{reader!s}")
207
203
  elif format == "toolkit":
208
- if reader.path.is_file():
209
- dms_importer = importers.DMSImporter.from_zip_file(reader.path)
210
- elif reader.path.is_dir():
211
- dms_importer = importers.DMSImporter.from_directory(reader.path)
212
- else:
213
- raise NeatValueError(f"Unsupported YAML format: {format}")
214
-
215
- ref_containers = dms_importer.root_schema.referenced_container()
216
- if system_container_ids := [
217
- container_id for container_id in ref_containers if container_id.space in COGNITE_SPACES
218
- ]:
219
- if self._client is None:
220
- raise NeatSessionError(
221
- "No client provided. You are referencing Cognite containers in your data model, "
222
- "NEAT needs a client to lookup the container definitions. "
223
- "Please set the client in the session, NeatSession(client=client)."
224
- )
225
- system_containers = self._client.loaders.containers.retrieve(system_container_ids)
226
- dms_importer.update_referenced_containers(system_containers)
227
-
204
+ dms_importer = importers.DMSImporter.from_path(path, self._client)
205
+ if dms_importer.issue_list.has_warning_type(MissingCogniteClientWarning):
206
+ raise NeatSessionError(
207
+ "No client provided. You are referencing Cognite containers in your data model, "
208
+ "NEAT needs a client to lookup the container definitions. "
209
+ "Please set the client in the session, NeatSession(client=client)."
210
+ )
228
211
  importer = dms_importer
229
212
  else:
230
213
  raise NeatValueError(f"Unsupported YAML format: {format}")
@@ -251,17 +234,9 @@ class CSVReadAPI(BaseReadAPI):
251
234
  """
252
235
 
253
236
  def __call__(self, io: Any, type: str, primary_key: str) -> None:
254
- reader = NeatReader.create(io)
255
- if isinstance(reader, HttpFileReader):
256
- path = Path(tempfile.gettempdir()).resolve() / reader.name
257
- path.write_text(reader.read_text(), encoding="utf-8", newline="\n")
258
- elif isinstance(reader, PathReader):
259
- path = reader.path
260
- else:
261
- raise NeatValueError("Only file paths are supported for CSV files")
262
237
  engine = import_engine()
263
238
  engine.set.format = "csv"
264
- engine.set.file = path
239
+ engine.set.file = NeatReader.create(io).materialize_path()
265
240
  engine.set.type = type
266
241
  engine.set.primary_key = primary_key
267
242
  extractor = engine.create_extractor()
@@ -283,14 +258,7 @@ class XMLReadAPI(BaseReadAPI):
283
258
  io: Any,
284
259
  format: XMLFileType | None = None,
285
260
  ) -> None:
286
- reader = NeatReader.create(io)
287
- if isinstance(reader, GitHubReader):
288
- path = Path(tempfile.gettempdir()).resolve() / reader.name
289
- path.write_text(reader.read_text())
290
- elif isinstance(reader, PathReader):
291
- path = reader.path
292
- else:
293
- raise NeatValueError("Only file paths are supported for XML files")
261
+ path = NeatReader.create(io).materialize_path()
294
262
  if format is None:
295
263
  format = xml_format_wizard()
296
264
 
@@ -303,7 +271,7 @@ class XMLReadAPI(BaseReadAPI):
303
271
  else:
304
272
  raise NeatValueError("Only support XML files of DEXPI format at the moment.")
305
273
 
306
- def dexpi(self, path):
274
+ def dexpi(self, io: Any) -> None:
307
275
  """Reads a DEXPI file into the NeatSession.
308
276
 
309
277
  Args:
@@ -314,13 +282,14 @@ class XMLReadAPI(BaseReadAPI):
314
282
  neat.read.xml.dexpi("url_or_path_to_dexpi_file")
315
283
  ```
316
284
  """
285
+ path = NeatReader.create(io).materialize_path()
317
286
  engine = import_engine()
318
287
  engine.set.format = "dexpi"
319
288
  engine.set.file = path
320
289
  extractor = engine.create_extractor()
321
290
  self._state.instances.store.write(extractor)
322
291
 
323
- def aml(self, path):
292
+ def aml(self, io: Any):
324
293
  """Reads an AML file into NeatSession.
325
294
 
326
295
  Args:
@@ -331,6 +300,7 @@ class XMLReadAPI(BaseReadAPI):
331
300
  neat.read.xml.aml("url_or_path_to_aml_file")
332
301
  ```
333
302
  """
303
+ path = NeatReader.create(io).materialize_path()
334
304
  engine = import_engine()
335
305
  engine.set.format = "aml"
336
306
  engine.set.file = path
@@ -362,9 +332,7 @@ class RDFReadAPI(BaseReadAPI):
362
332
  ```
363
333
  """
364
334
  reader = NeatReader.create(io)
365
- if not isinstance(reader, PathReader):
366
- raise NeatValueError("Only file paths are supported for RDF files")
367
- importer = importers.OWLImporter.from_file(reader.path, source_name=f"file {reader!s}")
335
+ importer = importers.OWLImporter.from_file(reader.materialize_path(), source_name=f"file {reader!s}")
368
336
  return self._state.rule_import(importer)
369
337
 
370
338
  def imf(self, io: Any) -> IssueList:
@@ -379,9 +347,7 @@ class RDFReadAPI(BaseReadAPI):
379
347
  ```
380
348
  """
381
349
  reader = NeatReader.create(io)
382
- if not isinstance(reader, PathReader):
383
- raise NeatValueError("Only file paths are supported for RDF files")
384
- importer = importers.IMFImporter.from_file(reader.path, source_name=f"file {reader!s}")
350
+ importer = importers.IMFImporter.from_file(reader.materialize_path(), source_name=f"file {reader!s}")
385
351
  return self._state.rule_import(importer)
386
352
 
387
353
  def __call__(
@@ -408,10 +374,7 @@ class RDFReadAPI(BaseReadAPI):
408
374
 
409
375
  elif type == "instances":
410
376
  reader = NeatReader.create(io)
411
- if not isinstance(reader, PathReader):
412
- raise NeatValueError("Only file paths are supported for RDF files")
413
-
414
- self._state.instances.store.write(extractors.RdfFileExtractor(reader.path))
377
+ self._state.instances.store.write(extractors.RdfFileExtractor(reader.materialize_path()))
415
378
  return IssueList()
416
379
  else:
417
380
  raise NeatSessionError(f"Expected data model or instances, got {type}")
@@ -4,7 +4,7 @@ from typing import Literal, cast
4
4
  from cognite.neat._issues import IssueList
5
5
  from cognite.neat._rules.importers import BaseImporter, InferenceImporter
6
6
  from cognite.neat._rules.models import DMSRules, InformationRules
7
- from cognite.neat._rules.transformers import RulesTransformer, ToExtension
7
+ from cognite.neat._rules.transformers import RulesTransformer, ToExtensionModel
8
8
  from cognite.neat._store import NeatGraphStore, NeatRulesStore
9
9
  from cognite.neat._store._rules_store import ModelEntity
10
10
  from cognite.neat._utils.rdf_ import uri_display_name
@@ -36,7 +36,7 @@ class SessionState:
36
36
  f"Moving back {len(pruned)} {step_str} to the last {location}."
37
37
  )
38
38
  if (
39
- any(isinstance(t, ToExtension) for t in transformer)
39
+ any(isinstance(t, ToExtensionModel) for t in transformer)
40
40
  and isinstance(self.rule_store.provenance[-1].target_entity, ModelEntity)
41
41
  and isinstance(self.rule_store.provenance[-1].target_entity.result, DMSRules | InformationRules)
42
42
  ):