cognite-neat 0.103.1__py3-none-any.whl → 0.105.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (175) hide show
  1. cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
  2. cognite/neat/_client/_api/schema.py +2 -1
  3. cognite/neat/_client/data_classes/neat_sequence.py +261 -0
  4. cognite/neat/_client/data_classes/schema.py +5 -1
  5. cognite/neat/_client/testing.py +33 -0
  6. cognite/neat/_constants.py +56 -0
  7. cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
  8. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
  9. cognite/neat/_graph/extractors/_mock_graph_generator.py +2 -2
  10. cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
  11. cognite/neat/_graph/transformers/__init__.py +3 -1
  12. cognite/neat/_graph/transformers/_base.py +109 -1
  13. cognite/neat/_graph/transformers/_classic_cdf.py +6 -1
  14. cognite/neat/_graph/transformers/_prune_graph.py +103 -47
  15. cognite/neat/_graph/transformers/_rdfpath.py +41 -17
  16. cognite/neat/_graph/transformers/_value_type.py +188 -151
  17. cognite/neat/_issues/__init__.py +0 -2
  18. cognite/neat/_issues/_base.py +54 -43
  19. cognite/neat/_issues/warnings/__init__.py +4 -1
  20. cognite/neat/_issues/warnings/_general.py +7 -0
  21. cognite/neat/_issues/warnings/_resources.py +12 -1
  22. cognite/neat/_rules/_shared.py +18 -34
  23. cognite/neat/_rules/exporters/_base.py +28 -2
  24. cognite/neat/_rules/exporters/_rules2dms.py +39 -1
  25. cognite/neat/_rules/exporters/_rules2excel.py +13 -2
  26. cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
  27. cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
  28. cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
  29. cognite/neat/_rules/importers/_base.py +9 -0
  30. cognite/neat/_rules/importers/_dms2rules.py +80 -57
  31. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
  32. cognite/neat/_rules/importers/_rdf/_base.py +10 -8
  33. cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
  34. cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
  35. cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
  36. cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
  37. cognite/neat/_rules/importers/_yaml2rules.py +21 -7
  38. cognite/neat/_rules/models/_base_input.py +1 -1
  39. cognite/neat/_rules/models/_base_rules.py +9 -1
  40. cognite/neat/_rules/models/dms/_rules.py +4 -0
  41. cognite/neat/_rules/models/dms/_rules_input.py +9 -0
  42. cognite/neat/_rules/models/entities/_wrapped.py +10 -5
  43. cognite/neat/_rules/models/information/_rules.py +4 -0
  44. cognite/neat/_rules/models/information/_rules_input.py +9 -0
  45. cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
  46. cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
  47. cognite/neat/_rules/transformers/__init__.py +13 -6
  48. cognite/neat/_rules/transformers/_base.py +41 -65
  49. cognite/neat/_rules/transformers/_converters.py +404 -234
  50. cognite/neat/_rules/transformers/_mapping.py +93 -72
  51. cognite/neat/_rules/transformers/_verification.py +50 -38
  52. cognite/neat/_session/_base.py +32 -121
  53. cognite/neat/_session/_inspect.py +5 -3
  54. cognite/neat/_session/_mapping.py +17 -105
  55. cognite/neat/_session/_prepare.py +138 -268
  56. cognite/neat/_session/_read.py +39 -195
  57. cognite/neat/_session/_set.py +6 -30
  58. cognite/neat/_session/_show.py +40 -21
  59. cognite/neat/_session/_state.py +49 -107
  60. cognite/neat/_session/_to.py +44 -33
  61. cognite/neat/_shared.py +23 -2
  62. cognite/neat/_store/_provenance.py +3 -82
  63. cognite/neat/_store/_rules_store.py +368 -10
  64. cognite/neat/_store/exceptions.py +23 -0
  65. cognite/neat/_utils/graph_transformations_report.py +36 -0
  66. cognite/neat/_utils/rdf_.py +8 -0
  67. cognite/neat/_utils/reader/_base.py +27 -0
  68. cognite/neat/_utils/spreadsheet.py +5 -4
  69. cognite/neat/_version.py +1 -1
  70. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
  71. cognite_neat-0.105.0.dist-info/RECORD +179 -0
  72. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
  73. cognite/neat/_app/api/__init__.py +0 -0
  74. cognite/neat/_app/api/asgi/metrics.py +0 -4
  75. cognite/neat/_app/api/configuration.py +0 -98
  76. cognite/neat/_app/api/context_manager/__init__.py +0 -3
  77. cognite/neat/_app/api/context_manager/manager.py +0 -16
  78. cognite/neat/_app/api/data_classes/__init__.py +0 -0
  79. cognite/neat/_app/api/data_classes/rest.py +0 -59
  80. cognite/neat/_app/api/explorer.py +0 -66
  81. cognite/neat/_app/api/routers/configuration.py +0 -25
  82. cognite/neat/_app/api/routers/crud.py +0 -102
  83. cognite/neat/_app/api/routers/metrics.py +0 -10
  84. cognite/neat/_app/api/routers/workflows.py +0 -224
  85. cognite/neat/_app/api/utils/__init__.py +0 -0
  86. cognite/neat/_app/api/utils/data_mapping.py +0 -17
  87. cognite/neat/_app/api/utils/logging.py +0 -26
  88. cognite/neat/_app/api/utils/query_templates.py +0 -92
  89. cognite/neat/_app/main.py +0 -17
  90. cognite/neat/_app/monitoring/__init__.py +0 -0
  91. cognite/neat/_app/monitoring/metrics.py +0 -69
  92. cognite/neat/_app/ui/index.html +0 -1
  93. cognite/neat/_app/ui/neat-app/.gitignore +0 -23
  94. cognite/neat/_app/ui/neat-app/README.md +0 -70
  95. cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
  96. cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
  97. cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
  98. cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
  99. cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
  100. cognite/neat/_app/ui/neat-app/build/index.html +0 -1
  101. cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
  102. cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
  103. cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
  104. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
  105. cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
  106. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
  107. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
  108. cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
  109. cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
  110. cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
  111. cognite/neat/_app/ui/neat-app/package.json +0 -62
  112. cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
  113. cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
  114. cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
  115. cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
  116. cognite/neat/_app/ui/neat-app/public/index.html +0 -43
  117. cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
  118. cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
  119. cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
  120. cognite/neat/_app/ui/neat-app/src/App.css +0 -38
  121. cognite/neat/_app/ui/neat-app/src/App.js +0 -17
  122. cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
  123. cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
  124. cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
  125. cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
  126. cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
  127. cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
  128. cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
  129. cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
  130. cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
  131. cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
  132. cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
  133. cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
  134. cognite/neat/_app/ui/neat-app/src/index.css +0 -13
  135. cognite/neat/_app/ui/neat-app/src/index.js +0 -13
  136. cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
  137. cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
  138. cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
  139. cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
  140. cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
  141. cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
  142. cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
  143. cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
  144. cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
  145. cognite/neat/_rules/transformers/_pipelines.py +0 -70
  146. cognite/neat/_workflows/__init__.py +0 -17
  147. cognite/neat/_workflows/base.py +0 -590
  148. cognite/neat/_workflows/cdf_store.py +0 -393
  149. cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
  150. cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
  151. cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
  152. cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
  153. cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  154. cognite/neat/_workflows/manager.py +0 -292
  155. cognite/neat/_workflows/model.py +0 -203
  156. cognite/neat/_workflows/steps/__init__.py +0 -0
  157. cognite/neat/_workflows/steps/data_contracts.py +0 -109
  158. cognite/neat/_workflows/steps/lib/__init__.py +0 -0
  159. cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
  160. cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
  161. cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
  162. cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
  163. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
  164. cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -398
  165. cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
  166. cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
  167. cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
  168. cognite/neat/_workflows/steps/step_model.py +0 -79
  169. cognite/neat/_workflows/steps_registry.py +0 -218
  170. cognite/neat/_workflows/tasks.py +0 -18
  171. cognite/neat/_workflows/triggers.py +0 -169
  172. cognite/neat/_workflows/utils.py +0 -19
  173. cognite_neat-0.103.1.dist-info/RECORD +0 -275
  174. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
  175. {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
@@ -5,17 +5,18 @@ from urllib.parse import quote
5
5
 
6
6
  import rdflib
7
7
  from rdflib import RDF, XSD, Graph, Namespace, URIRef
8
+ from rdflib.query import ResultRow
8
9
 
9
10
  from cognite.neat._constants import UNKNOWN_TYPE
10
11
  from cognite.neat._graph.queries import Queries
11
- from cognite.neat._issues.warnings import NeatValueWarning, PropertyDataTypeConversionWarning
12
+ from cognite.neat._issues.warnings import PropertyDataTypeConversionWarning
12
13
  from cognite.neat._utils.auxiliary import string_to_ideal_type
13
- from cognite.neat._utils.collection_ import iterate_progress_bar
14
14
  from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
15
15
 
16
- from ._base import BaseTransformer
16
+ from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
17
17
 
18
18
 
19
+ # TODO: Standardise
19
20
  class SplitMultiValueProperty(BaseTransformer):
20
21
  description: str = (
21
22
  "SplitMultiValueProperty is a transformer that splits a "
@@ -75,34 +76,11 @@ class SplitMultiValueProperty(BaseTransformer):
75
76
  graph.add((s, new_property, o))
76
77
 
77
78
 
78
- class ConvertLiteral(BaseTransformer):
79
+ class ConvertLiteral(BaseTransformerStandardised):
79
80
  description: str = "ConvertLiteral is a transformer that improve data typing of a literal value."
80
81
  _use_only_once: bool = False
81
82
  _need_changes = frozenset({})
82
83
 
83
- _count_by_properties = """SELECT (COUNT(?value) AS ?valueCount)
84
- WHERE {{
85
- ?instance a <{subject_type}> .
86
- ?instance <{subject_predicate}> ?value
87
- FILTER(isLiteral(?value))
88
- }}"""
89
-
90
- _count_by_properties_uri = """SELECT (COUNT(?value) AS ?valueCount)
91
- WHERE {{
92
- ?instance a <{subject_type}> .
93
- ?instance <{subject_predicate}> ?value
94
- FILTER(isIRI(?value))
95
- }}"""
96
-
97
- _properties = """SELECT ?instance ?value
98
- WHERE {{
99
- ?instance a <{subject_type}> .
100
- ?instance <{subject_predicate}> ?value
101
-
102
- FILTER(isLiteral(?value))
103
-
104
- }}"""
105
-
106
84
  def __init__(
107
85
  self,
108
86
  subject_type: URIRef,
@@ -115,89 +93,55 @@ class ConvertLiteral(BaseTransformer):
115
93
  self._type_name = remove_namespace_from_uri(subject_type)
116
94
  self._property_name = remove_namespace_from_uri(subject_predicate)
117
95
 
118
- def transform(self, graph: Graph) -> None:
119
- count_connection_query = self._count_by_properties_uri.format(
120
- subject_type=self.subject_type, subject_predicate=self.subject_predicate
121
- )
122
- connection_count_res = list(graph.query(count_connection_query))
123
- connection_count = int(connection_count_res[0][0]) # type: ignore [index, arg-type]
124
-
125
- if connection_count > 0:
96
+ def _skip_count_query(self) -> str:
97
+ query = """SELECT (COUNT(?value) AS ?valueCount)
98
+ WHERE {{
99
+ ?instance a <{subject_type}> .
100
+ ?instance <{subject_predicate}> ?value
101
+ FILTER(isIRI(?value))
102
+ }}"""
103
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
104
+
105
+ def _count_query(self) -> str:
106
+ query = """SELECT (COUNT(?value) AS ?valueCount)
107
+ WHERE {{
108
+ ?instance a <{subject_type}> .
109
+ ?instance <{subject_predicate}> ?value
110
+ FILTER(isLiteral(?value))
111
+ }}"""
112
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
113
+
114
+ def _iterate_query(self) -> str:
115
+ query = """SELECT ?instance ?value
116
+ WHERE {{
117
+ ?instance a <{subject_type}> .
118
+ ?instance <{subject_predicate}> ?value
119
+ FILTER(isLiteral(?value))
120
+ }}"""
121
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
122
+
123
+ def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
124
+ row_output = RowTransformationOutput()
125
+
126
+ instance, literal = query_result_row
127
+ value = cast(rdflib.Literal, literal).toPython()
128
+
129
+ try:
130
+ converted_value = self.conversion(value)
131
+ except Exception as e:
126
132
  warnings.warn(
127
- NeatValueWarning(
128
- f"Skipping {connection_count} of {self._type_name}.{self._property_name} "
129
- f"as these are connections and not data values."
130
- ),
133
+ PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
131
134
  stacklevel=2,
132
135
  )
136
+ row_output.add_triples.append((instance, self.subject_predicate, rdflib.Literal(converted_value))) # type: ignore[arg-type]
137
+ row_output.remove_triples.append((instance, self.subject_predicate, literal)) # type: ignore[arg-type]
138
+ row_output.instances_modified_count += 1
133
139
 
134
- count_query = self._count_by_properties.format(
135
- subject_type=self.subject_type, subject_predicate=self.subject_predicate
136
- )
137
-
138
- property_count_res = list(graph.query(count_query))
139
- property_count = int(property_count_res[0][0]) # type: ignore [index, arg-type]
140
- iterate_query = self._properties.format(
141
- subject_type=self.subject_type, subject_predicate=self.subject_predicate
142
- )
143
-
144
- for instance, literal in iterate_progress_bar( # type: ignore[misc]
145
- graph.query(iterate_query),
146
- total=property_count,
147
- description=f"Converting {self._type_name}.{self._property_name}.",
148
- ):
149
- value = cast(rdflib.Literal, literal).toPython()
150
- try:
151
- converted_value = self.conversion(value)
152
- except Exception as e:
153
- warnings.warn(
154
- PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
155
- stacklevel=2,
156
- )
157
- continue
158
-
159
- graph.add((instance, self.subject_predicate, rdflib.Literal(converted_value)))
160
- graph.remove((instance, self.subject_predicate, literal))
161
-
162
-
163
- class LiteralToEntity(BaseTransformer):
164
- description = "Converts a literal value to new entity"
140
+ return row_output
165
141
 
166
- _count_properties_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
167
- WHERE {{
168
- ?instance a <{subject_type}> .
169
- ?instance <{subject_predicate}> ?property
170
- FILTER(isLiteral(?property))
171
- }}"""
172
- _count_connections_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
173
- WHERE {{
174
- ?instance a <{subject_type}> .
175
- ?instance <{subject_predicate}> ?property
176
- FILTER(isIRI(?property))
177
- }}"""
178
-
179
- _properties_of_type = """SELECT ?instance ?property
180
- WHERE {{
181
- ?instance a <{subject_type}> .
182
- ?instance <{subject_predicate}> ?property
183
- FILTER(isLiteral(?property))
184
- }}"""
185
-
186
- _count_properties = """SELECT (COUNT(?property) AS ?propertyCount)
187
- WHERE {{
188
- ?instance <{subject_predicate}> ?property
189
- FILTER(isLiteral(?property))
190
- }}"""
191
- _count_connections = """SELECT (COUNT(?property) AS ?propertyCount)
192
- WHERE {{
193
- ?instance <{subject_predicate}> ?property
194
- FILTER(isIRI(?property))
195
- }}"""
196
- _properties = """SELECT ?instance ?property
197
- WHERE {{
198
- ?instance <{subject_predicate}> ?property
199
- FILTER(isLiteral(?property))
200
- }}"""
142
+
143
+ class LiteralToEntity(BaseTransformerStandardised):
144
+ description = "Converts a literal value to new entity"
201
145
 
202
146
  def __init__(
203
147
  self, subject_type: URIRef | None, subject_predicate: URIRef, entity_type: str, new_property: str | None = None
@@ -207,54 +151,147 @@ class LiteralToEntity(BaseTransformer):
207
151
  self.entity_type = entity_type
208
152
  self.new_property = new_property
209
153
 
210
- def transform(self, graph: Graph) -> None:
154
+ def _iterate_query(self) -> str:
211
155
  if self.subject_type is None:
212
- count_query = self._count_properties.format(subject_predicate=self.subject_predicate)
213
- iterate_query = self._properties.format(subject_predicate=self.subject_predicate)
214
- connection_count_query = self._count_connections.format(subject_predicate=self.subject_predicate)
156
+ query = """SELECT ?instance ?property
157
+ WHERE {{
158
+ ?instance <{subject_predicate}> ?property
159
+ FILTER(isLiteral(?property))
160
+ }}"""
161
+ return query.format(subject_predicate=self.subject_predicate)
215
162
  else:
216
- count_query = self._count_properties_of_type.format(
217
- subject_type=self.subject_type, subject_predicate=self.subject_predicate
218
- )
219
- iterate_query = self._properties_of_type.format(
220
- subject_type=self.subject_type, subject_predicate=self.subject_predicate
221
- )
222
- connection_count_query = self._count_connections_of_type.format(
223
- subject_type=self.subject_type, subject_predicate=self.subject_predicate
224
- )
163
+ query = """SELECT ?instance ?property
164
+ WHERE {{
165
+ ?instance a <{subject_type}> .
166
+ ?instance <{subject_predicate}> ?property
167
+ FILTER(isLiteral(?property))
168
+ }}"""
169
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
170
+
171
+ def _skip_count_query(self) -> str:
172
+ if self.subject_type is None:
173
+ query = """SELECT (COUNT(?property) AS ?propertyCount)
174
+ WHERE {{
175
+ ?instance <{subject_predicate}> ?property
176
+ FILTER(isIRI(?property))
177
+ }}"""
178
+ return query.format(subject_predicate=self.subject_predicate)
179
+ else:
180
+ query = """SELECT (COUNT(?property) AS ?propertyCount)
181
+ WHERE {{
182
+ ?instance a <{subject_type}> .
183
+ ?instance <{subject_predicate}> ?property
184
+ FILTER(isIRI(?property))
185
+ }}"""
186
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
187
+
188
+ def _count_query(self) -> str:
189
+ if self.subject_type is None:
190
+ query = """SELECT (COUNT(?property) AS ?propertyCount)
191
+ WHERE {{
192
+ ?instance <{subject_predicate}> ?property
193
+ FILTER(isLiteral(?property))
194
+ }}"""
195
+ return query.format(subject_predicate=self.subject_predicate)
196
+ else:
197
+ query = """SELECT (COUNT(?property) AS ?propertyCount)
198
+ WHERE {{
199
+ ?instance a <{subject_type}> .
200
+ ?instance <{subject_predicate}> ?property
201
+ FILTER(isLiteral(?property))
202
+ }}"""
225
203
 
226
- connection_count_res = list(graph.query(connection_count_query))
227
- connection_count = int(connection_count_res[0][0]) # type: ignore [index, arg-type]
228
- if connection_count > 0:
229
- warnings.warn(
230
- NeatValueWarning(
231
- f"Skipping {connection_count} of {remove_namespace_from_uri(self.subject_predicate)} "
232
- f"as these are connections and not data values."
233
- ),
234
- stacklevel=2,
235
- )
204
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
236
205
 
237
- property_count_res = list(graph.query(count_query))
238
- property_count = int(property_count_res[0][0]) # type: ignore [index, arg-type]
206
+ def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
207
+ row_output = RowTransformationOutput()
239
208
 
240
- instance: URIRef
241
- description = f"Creating {remove_namespace_from_uri(self.subject_predicate)}."
242
- if self.subject_type is not None:
243
- description = (
244
- f"Creating {remove_namespace_from_uri(self.subject_type)}."
245
- f"{remove_namespace_from_uri(self.subject_predicate)}."
246
- )
247
- for instance, literal in iterate_progress_bar( # type: ignore[misc, assignment]
248
- graph.query(iterate_query),
249
- total=property_count,
250
- description=description,
251
- ):
252
- value = cast(rdflib.Literal, literal).toPython()
253
- namespace = Namespace(get_namespace(instance))
254
- entity_type = namespace[self.entity_type]
255
- new_entity = namespace[f"{self.entity_type}_{quote(value)!s}"]
256
- graph.add((new_entity, RDF.type, entity_type))
257
- if self.new_property is not None:
258
- graph.add((new_entity, namespace[self.new_property], rdflib.Literal(value)))
259
- graph.add((instance, self.subject_predicate, new_entity))
260
- graph.remove((instance, self.subject_predicate, literal))
209
+ instance, literal = query_result_row
210
+ value = cast(rdflib.Literal, literal).toPython()
211
+ namespace = Namespace(get_namespace(instance)) # type: ignore[arg-type]
212
+ entity_type = namespace[self.entity_type]
213
+ new_entity = namespace[f"{self.entity_type}_{quote(value)!s}"]
214
+ row_output.add_triples.append((new_entity, RDF.type, entity_type))
215
+ row_output.instances_added_count += 1 # we add one new entity
216
+
217
+ if self.new_property is not None:
218
+ row_output.add_triples.append((new_entity, namespace[self.new_property], rdflib.Literal(value))) # type: ignore[arg-type]
219
+ row_output.instances_modified_count += 1 # we modify the new entity
220
+
221
+ row_output.add_triples.append((instance, self.subject_predicate, new_entity)) # type: ignore[arg-type]
222
+ row_output.remove_triples.append((instance, self.subject_predicate, literal)) # type: ignore[arg-type]
223
+ row_output.instances_modified_count += 1 # we modify the old entity
224
+
225
+ return row_output
226
+
227
+
228
+ class ConnectionToLiteral(BaseTransformerStandardised):
229
+ description = "Converts an entity connection to a literal value"
230
+
231
+ def __init__(self, subject_type: URIRef | None, subject_predicate: URIRef) -> None:
232
+ self.subject_type = subject_type
233
+ self.subject_predicate = subject_predicate
234
+
235
+ def _iterate_query(self) -> str:
236
+ if self.subject_type is None:
237
+ query = """SELECT ?instance ?object
238
+ WHERE {{
239
+ ?instance <{subject_predicate}> ?object
240
+ FILTER(isIRI(?object))
241
+ }}"""
242
+ return query.format(subject_predicate=self.subject_predicate)
243
+ else:
244
+ query = """SELECT ?instance ?object
245
+ WHERE {{
246
+ ?instance a <{subject_type}> .
247
+ ?instance <{subject_predicate}> ?object
248
+ FILTER(isIRI(?object))
249
+ }}"""
250
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
251
+
252
+ def _skip_count_query(self) -> str:
253
+ if self.subject_type is None:
254
+ query = """SELECT (COUNT(?object) AS ?objectCount)
255
+ WHERE {{
256
+ ?instance <{subject_predicate}> ?object
257
+ FILTER(isLiteral(?object))
258
+ }}"""
259
+ return query.format(subject_predicate=self.subject_predicate)
260
+ else:
261
+ query = """SELECT (COUNT(?object) AS ?objectCount)
262
+ WHERE {{
263
+ ?instance a <{subject_type}> .
264
+ ?instance <{subject_predicate}> ?object
265
+ FILTER(isLiteral(?object))
266
+ }}"""
267
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
268
+
269
+ def _count_query(self) -> str:
270
+ if self.subject_type is None:
271
+ query = """SELECT (COUNT(?object) AS ?objectCount)
272
+ WHERE {{
273
+ ?instance <{subject_predicate}> ?object
274
+ FILTER(isIRI(?object))
275
+ }}"""
276
+ return query.format(subject_predicate=self.subject_predicate)
277
+ else:
278
+ query = """SELECT (COUNT(?object) AS ?objectCount)
279
+ WHERE {{
280
+ ?instance a <{subject_type}> .
281
+ ?instance <{subject_predicate}> ?object
282
+ FILTER(isIRI(?object))
283
+ }}"""
284
+
285
+ return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
286
+
287
+ def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
288
+ row_output = RowTransformationOutput()
289
+
290
+ instance, object_entity = cast(tuple[URIRef, URIRef], query_result_row)
291
+ value = remove_namespace_from_uri(object_entity)
292
+
293
+ row_output.add_triples.append((instance, self.subject_predicate, rdflib.Literal(value)))
294
+ row_output.remove_triples.append((instance, self.subject_predicate, object_entity))
295
+ row_output.instances_modified_count += 1
296
+
297
+ return row_output
@@ -3,7 +3,6 @@ as some helper classes to handle them like NeatIssueList"""
3
3
 
4
4
  from ._base import (
5
5
  DefaultWarning,
6
- FutureResult,
7
6
  IssueList,
8
7
  MultiValueError,
9
8
  NeatError,
@@ -16,7 +15,6 @@ from ._base import (
16
15
 
17
16
  __all__ = [
18
17
  "DefaultWarning",
19
- "FutureResult",
20
18
  "IssueList",
21
19
  "MultiValueError",
22
20
  "NeatError",
@@ -390,11 +390,16 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
390
390
  """This is a generic list of NeatIssues."""
391
391
 
392
392
  def __init__(
393
- self, issues: Sequence[T_NeatIssue] | None = None, title: str | None = None, action: str | None = None
393
+ self,
394
+ issues: Sequence[T_NeatIssue] | None = None,
395
+ title: str | None = None,
396
+ action: str | None = None,
397
+ hint: str | None = None,
394
398
  ):
395
399
  super().__init__(issues or [])
396
400
  self.title = title
397
401
  self.action = action
402
+ self.hint = hint
398
403
 
399
404
  @property
400
405
  def errors(self) -> Self:
@@ -406,6 +411,11 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
406
411
  """Return True if this list contains any errors."""
407
412
  return any(isinstance(issue, NeatError) for issue in self)
408
413
 
414
+ @property
415
+ def has_warnings(self) -> bool:
416
+ """Return True if this list contains any warnings."""
417
+ return any(isinstance(issue, NeatWarning) for issue in self)
418
+
409
419
  @property
410
420
  def warnings(self) -> Self:
411
421
  """Return all the warnings in this list."""
@@ -415,6 +425,10 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
415
425
  """Return True if this list contains any errors of the given type."""
416
426
  return any(isinstance(issue, error_type) for issue in self)
417
427
 
428
+ def has_warning_type(self, warning_type: type[NeatWarning]) -> bool:
429
+ """Return True if this list contains any warnings of the given type."""
430
+ return any(isinstance(issue, warning_type) for issue in self)
431
+
418
432
  def as_errors(self, operation: str = "Operation failed") -> ExceptionGroup:
419
433
  """Return an ExceptionGroup with all the errors in this list."""
420
434
  return ExceptionGroup(
@@ -455,14 +469,31 @@ class IssueList(NeatIssueList[NeatIssue]):
455
469
  """This is a list of NeatIssues."""
456
470
 
457
471
  def _repr_html_(self) -> str | None:
458
- if not self:
459
- return "<p>'No issues found'</p>"
460
- df = self.to_pandas()
461
- agg_df = df["NeatIssue"].value_counts().to_frame()
462
- if len(agg_df) < 10:
463
- return agg_df._repr_html_() # type: ignore[operator]
472
+ if self.action and not self:
473
+ header = f"Success: {self.action}"
474
+ elif self.action and self.has_errors:
475
+ header = f"Failed: {self.action}"
476
+ elif self.action and self.has_warnings:
477
+ header = f"Succeeded with warnings: {self.action}"
478
+ elif not self:
479
+ header = "No issues found"
464
480
  else:
465
- return agg_df.head()._repr_html_() # type: ignore[operator]
481
+ header = ""
482
+
483
+ body = f"<p>{header}</p>"
484
+
485
+ if self:
486
+ df = self.to_pandas()
487
+ agg_df = df["NeatIssue"].value_counts().to_frame()
488
+ if len(agg_df) < 10:
489
+ table = agg_df._repr_html_() # type: ignore[operator]
490
+ else:
491
+ table = agg_df.head()._repr_html_() # type: ignore[operator]
492
+ body += f"<br />{table}"
493
+
494
+ if self.hint:
495
+ body += f"<br />Hint: {self.hint}"
496
+ return body
466
497
 
467
498
 
468
499
  T_Cls = TypeVar("T_Cls")
@@ -478,55 +509,35 @@ def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[t
478
509
 
479
510
 
480
511
  @contextmanager
481
- def catch_warnings(
482
- issues: IssueList | None = None,
483
- warning_cls: type[NeatWarning] = DefaultWarning,
484
- ) -> Iterator[None]:
512
+ def catch_warnings() -> Iterator[IssueList]:
485
513
  """Catch warnings and append them to the issues list."""
514
+ issues = IssueList()
486
515
  with warnings.catch_warnings(record=True) as warning_logger:
487
516
  warnings.simplefilter("always")
488
517
  try:
489
- yield None
518
+ yield issues
490
519
  finally:
491
- if warning_logger and issues is not None:
492
- issues.extend([warning_cls.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
493
-
494
-
495
- class FutureResult:
496
- def __init__(self) -> None:
497
- self._result: Literal["success", "failure", "pending"] = "pending"
498
-
499
- @property
500
- def result(self) -> Literal["success", "failure", "pending"]:
501
- return self._result
520
+ if warning_logger:
521
+ issues.extend([NeatWarning.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
502
522
 
503
523
 
504
524
  @contextmanager
505
- def catch_issues(
506
- issues: IssueList,
507
- error_cls: type[NeatError] = NeatError,
508
- warning_cls: type[NeatWarning] = NeatWarning,
509
- error_args: dict[str, Any] | None = None,
510
- ) -> Iterator[FutureResult]:
525
+ def catch_issues(error_args: dict[str, Any] | None = None) -> Iterator[IssueList]:
511
526
  """This is an internal help function to handle issues and warnings.
512
527
 
513
528
  Args:
514
- issues: The issues list to append to.
515
- error_cls: The class used to convert errors to issues.
516
- warning_cls: The class used to convert warnings to issues.
529
+ error_args: Additional arguments to pass to the error class. The only use case as of (2025-01-03) is to pass
530
+ the read_info_by_sheet to the error class such that the row numbers can be adjusted to match the source
531
+ spreadsheet.
517
532
 
518
533
  Returns:
519
- FutureResult: A future result object that can be used to check the result of the context manager.
534
+ IssueList: The list of issues.
535
+
520
536
  """
521
- with catch_warnings(issues, warning_cls):
522
- future_result = FutureResult()
537
+ with catch_warnings() as issues:
523
538
  try:
524
- yield future_result
539
+ yield issues
525
540
  except ValidationError as e:
526
- issues.extend(error_cls.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
527
- future_result._result = "failure"
541
+ issues.extend(NeatError.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
528
542
  except (NeatError, MultiValueError) as e:
529
- issues.extend(error_cls.from_errors([e], **(error_args or {}))) # type: ignore[arg-type, list-item]
530
- future_result._result = "failure"
531
- else:
532
- future_result._result = "success"
543
+ issues.extend(NeatError.from_errors([e], **(error_args or {}))) # type: ignore[arg-type, list-item]
@@ -13,7 +13,7 @@ from ._external import (
13
13
  FileReadWarning,
14
14
  FileTypeUnexpectedWarning,
15
15
  )
16
- from ._general import NeatValueWarning, NotSupportedWarning, RegexViolationWarning
16
+ from ._general import MissingCogniteClientWarning, NeatValueWarning, NotSupportedWarning, RegexViolationWarning
17
17
  from ._models import (
18
18
  BreakingModelingPrincipleWarning,
19
19
  CDFNotSupportedWarning,
@@ -41,6 +41,7 @@ from ._resources import (
41
41
  ResourceRetrievalWarning,
42
42
  ResourcesDuplicatedWarning,
43
43
  ResourceTypeNotSupportedWarning,
44
+ ResourceUnknownWarning,
44
45
  )
45
46
 
46
47
  __all__ = [
@@ -53,6 +54,7 @@ __all__ = [
53
54
  "FileMissingRequiredFieldWarning",
54
55
  "FileReadWarning",
55
56
  "FileTypeUnexpectedWarning",
57
+ "MissingCogniteClientWarning",
56
58
  "NeatValueWarning",
57
59
  "NotSupportedHasDataFilterLimitWarning",
58
60
  "NotSupportedViewContainerLimitWarning",
@@ -73,6 +75,7 @@ __all__ = [
73
75
  "ResourceRegexViolationWarning",
74
76
  "ResourceRetrievalWarning",
75
77
  "ResourceTypeNotSupportedWarning",
78
+ "ResourceUnknownWarning",
76
79
  "ResourcesDuplicatedWarning",
77
80
  "UndefinedViewWarning",
78
81
  "UserModelingWarning",
@@ -27,3 +27,10 @@ class RegexViolationWarning(NeatWarning):
27
27
  identifier: str
28
28
  pattern_name: str
29
29
  motivation: str | None = None
30
+
31
+
32
+ @dataclass(unsafe_hash=True)
33
+ class MissingCogniteClientWarning(NeatWarning):
34
+ """Missing Cognite Client required for {functionality}"""
35
+
36
+ functionality: str
@@ -21,7 +21,7 @@ class ResourceRegexViolationWarning(ResourceNeatWarning):
21
21
 
22
22
  fix = (
23
23
  "Either export the data model and make the necessary changes manually"
24
- " or run prepare.cdf_compliant_external_ids."
24
+ " or run prepare.data_model.cdf_compliant_external_ids."
25
25
  )
26
26
 
27
27
  location: str
@@ -39,6 +39,17 @@ class ResourceNotFoundWarning(ResourceNeatWarning, Generic[T_Identifier, T_Refer
39
39
  referred_type: str
40
40
 
41
41
 
42
+ @dataclass(unsafe_hash=True)
43
+ class ResourceUnknownWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
44
+ """The {resource_type} with identifier {identifier} referred by {referred_type} {referred_by} is unknown.
45
+ Will continue, but the model is incomplete."""
46
+
47
+ referred_by: T_ReferenceIdentifier
48
+ referred_type: str
49
+
50
+ fix = "You can maybe retrieve the resource from the CDF."
51
+
52
+
42
53
  @dataclass(unsafe_hash=True)
43
54
  class ResourceNotDefinedWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
44
55
  """The {resource_type} {identifier} is not defined in the {location}"""