cognite-neat 0.103.1__py3-none-any.whl → 0.105.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/_api/data_modeling_loaders.py +83 -23
- cognite/neat/_client/_api/schema.py +2 -1
- cognite/neat/_client/data_classes/neat_sequence.py +261 -0
- cognite/neat/_client/data_classes/schema.py +5 -1
- cognite/neat/_client/testing.py +33 -0
- cognite/neat/_constants.py +56 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +6 -5
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +225 -11
- cognite/neat/_graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/_graph/loaders/_rdf2dms.py +13 -2
- cognite/neat/_graph/transformers/__init__.py +3 -1
- cognite/neat/_graph/transformers/_base.py +109 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +6 -1
- cognite/neat/_graph/transformers/_prune_graph.py +103 -47
- cognite/neat/_graph/transformers/_rdfpath.py +41 -17
- cognite/neat/_graph/transformers/_value_type.py +188 -151
- cognite/neat/_issues/__init__.py +0 -2
- cognite/neat/_issues/_base.py +54 -43
- cognite/neat/_issues/warnings/__init__.py +4 -1
- cognite/neat/_issues/warnings/_general.py +7 -0
- cognite/neat/_issues/warnings/_resources.py +12 -1
- cognite/neat/_rules/_shared.py +18 -34
- cognite/neat/_rules/exporters/_base.py +28 -2
- cognite/neat/_rules/exporters/_rules2dms.py +39 -1
- cognite/neat/_rules/exporters/_rules2excel.py +13 -2
- cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
- cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
- cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
- cognite/neat/_rules/importers/_base.py +9 -0
- cognite/neat/_rules/importers/_dms2rules.py +80 -57
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
- cognite/neat/_rules/importers/_rdf/_base.py +10 -8
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
- cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
- cognite/neat/_rules/importers/_yaml2rules.py +21 -7
- cognite/neat/_rules/models/_base_input.py +1 -1
- cognite/neat/_rules/models/_base_rules.py +9 -1
- cognite/neat/_rules/models/dms/_rules.py +4 -0
- cognite/neat/_rules/models/dms/_rules_input.py +9 -0
- cognite/neat/_rules/models/entities/_wrapped.py +10 -5
- cognite/neat/_rules/models/information/_rules.py +4 -0
- cognite/neat/_rules/models/information/_rules_input.py +9 -0
- cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
- cognite/neat/_rules/models/mapping/_classic2core.yaml +239 -38
- cognite/neat/_rules/transformers/__init__.py +13 -6
- cognite/neat/_rules/transformers/_base.py +41 -65
- cognite/neat/_rules/transformers/_converters.py +404 -234
- cognite/neat/_rules/transformers/_mapping.py +93 -72
- cognite/neat/_rules/transformers/_verification.py +50 -38
- cognite/neat/_session/_base.py +32 -121
- cognite/neat/_session/_inspect.py +5 -3
- cognite/neat/_session/_mapping.py +17 -105
- cognite/neat/_session/_prepare.py +138 -268
- cognite/neat/_session/_read.py +39 -195
- cognite/neat/_session/_set.py +6 -30
- cognite/neat/_session/_show.py +40 -21
- cognite/neat/_session/_state.py +49 -107
- cognite/neat/_session/_to.py +44 -33
- cognite/neat/_shared.py +23 -2
- cognite/neat/_store/_provenance.py +3 -82
- cognite/neat/_store/_rules_store.py +368 -10
- cognite/neat/_store/exceptions.py +23 -0
- cognite/neat/_utils/graph_transformations_report.py +36 -0
- cognite/neat/_utils/rdf_.py +8 -0
- cognite/neat/_utils/reader/_base.py +27 -0
- cognite/neat/_utils/spreadsheet.py +5 -4
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/METADATA +3 -2
- cognite_neat-0.105.0.dist-info/RECORD +179 -0
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/WHEEL +1 -1
- cognite/neat/_app/api/__init__.py +0 -0
- cognite/neat/_app/api/asgi/metrics.py +0 -4
- cognite/neat/_app/api/configuration.py +0 -98
- cognite/neat/_app/api/context_manager/__init__.py +0 -3
- cognite/neat/_app/api/context_manager/manager.py +0 -16
- cognite/neat/_app/api/data_classes/__init__.py +0 -0
- cognite/neat/_app/api/data_classes/rest.py +0 -59
- cognite/neat/_app/api/explorer.py +0 -66
- cognite/neat/_app/api/routers/configuration.py +0 -25
- cognite/neat/_app/api/routers/crud.py +0 -102
- cognite/neat/_app/api/routers/metrics.py +0 -10
- cognite/neat/_app/api/routers/workflows.py +0 -224
- cognite/neat/_app/api/utils/__init__.py +0 -0
- cognite/neat/_app/api/utils/data_mapping.py +0 -17
- cognite/neat/_app/api/utils/logging.py +0 -26
- cognite/neat/_app/api/utils/query_templates.py +0 -92
- cognite/neat/_app/main.py +0 -17
- cognite/neat/_app/monitoring/__init__.py +0 -0
- cognite/neat/_app/monitoring/metrics.py +0 -69
- cognite/neat/_app/ui/index.html +0 -1
- cognite/neat/_app/ui/neat-app/.gitignore +0 -23
- cognite/neat/_app/ui/neat-app/README.md +0 -70
- cognite/neat/_app/ui/neat-app/build/asset-manifest.json +0 -14
- cognite/neat/_app/ui/neat-app/build/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/build/img/architect-icon.svg +0 -116
- cognite/neat/_app/ui/neat-app/build/img/developer-icon.svg +0 -112
- cognite/neat/_app/ui/neat-app/build/img/sme-icon.svg +0 -34
- cognite/neat/_app/ui/neat-app/build/index.html +0 -1
- cognite/neat/_app/ui/neat-app/build/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/build/manifest.json +0 -25
- cognite/neat/_app/ui/neat-app/build/robots.txt +0 -3
- cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css +0 -2
- cognite/neat/_app/ui/neat-app/build/static/css/main.72e3d92e.css.map +0 -1
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js +0 -3
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.LICENSE.txt +0 -88
- cognite/neat/_app/ui/neat-app/build/static/js/main.5a52cf09.js.map +0 -1
- cognite/neat/_app/ui/neat-app/build/static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg +0 -1
- cognite/neat/_app/ui/neat-app/package-lock.json +0 -18306
- cognite/neat/_app/ui/neat-app/package.json +0 -62
- cognite/neat/_app/ui/neat-app/public/favicon.ico +0 -0
- cognite/neat/_app/ui/neat-app/public/img/architect-icon.svg +0 -116
- cognite/neat/_app/ui/neat-app/public/img/developer-icon.svg +0 -112
- cognite/neat/_app/ui/neat-app/public/img/sme-icon.svg +0 -34
- cognite/neat/_app/ui/neat-app/public/index.html +0 -43
- cognite/neat/_app/ui/neat-app/public/logo192.png +0 -0
- cognite/neat/_app/ui/neat-app/public/manifest.json +0 -25
- cognite/neat/_app/ui/neat-app/public/robots.txt +0 -3
- cognite/neat/_app/ui/neat-app/src/App.css +0 -38
- cognite/neat/_app/ui/neat-app/src/App.js +0 -17
- cognite/neat/_app/ui/neat-app/src/App.test.js +0 -8
- cognite/neat/_app/ui/neat-app/src/MainContainer.tsx +0 -70
- cognite/neat/_app/ui/neat-app/src/components/JsonViewer.tsx +0 -43
- cognite/neat/_app/ui/neat-app/src/components/LocalUploader.tsx +0 -124
- cognite/neat/_app/ui/neat-app/src/components/OverviewComponentEditorDialog.tsx +0 -63
- cognite/neat/_app/ui/neat-app/src/components/StepEditorDialog.tsx +0 -511
- cognite/neat/_app/ui/neat-app/src/components/TabPanel.tsx +0 -36
- cognite/neat/_app/ui/neat-app/src/components/Utils.tsx +0 -56
- cognite/neat/_app/ui/neat-app/src/components/WorkflowDeleteDialog.tsx +0 -60
- cognite/neat/_app/ui/neat-app/src/components/WorkflowExecutionReport.tsx +0 -112
- cognite/neat/_app/ui/neat-app/src/components/WorkflowImportExportDialog.tsx +0 -67
- cognite/neat/_app/ui/neat-app/src/components/WorkflowMetadataDialog.tsx +0 -79
- cognite/neat/_app/ui/neat-app/src/index.css +0 -13
- cognite/neat/_app/ui/neat-app/src/index.js +0 -13
- cognite/neat/_app/ui/neat-app/src/logo.svg +0 -1
- cognite/neat/_app/ui/neat-app/src/reportWebVitals.js +0 -13
- cognite/neat/_app/ui/neat-app/src/setupTests.js +0 -5
- cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts +0 -388
- cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx +0 -61
- cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx +0 -184
- cognite/neat/_app/ui/neat-app/src/views/GlobalConfigView.tsx +0 -180
- cognite/neat/_app/ui/neat-app/src/views/WorkflowView.tsx +0 -570
- cognite/neat/_app/ui/neat-app/tsconfig.json +0 -27
- cognite/neat/_rules/transformers/_pipelines.py +0 -70
- cognite/neat/_workflows/__init__.py +0 -17
- cognite/neat/_workflows/base.py +0 -590
- cognite/neat/_workflows/cdf_store.py +0 -393
- cognite/neat/_workflows/examples/Export_DMS/workflow.yaml +0 -89
- cognite/neat/_workflows/examples/Export_Semantic_Data_Model/workflow.yaml +0 -66
- cognite/neat/_workflows/examples/Import_DMS/workflow.yaml +0 -65
- cognite/neat/_workflows/examples/Validate_Rules/workflow.yaml +0 -67
- cognite/neat/_workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
- cognite/neat/_workflows/manager.py +0 -292
- cognite/neat/_workflows/model.py +0 -203
- cognite/neat/_workflows/steps/__init__.py +0 -0
- cognite/neat/_workflows/steps/data_contracts.py +0 -109
- cognite/neat/_workflows/steps/lib/__init__.py +0 -0
- cognite/neat/_workflows/steps/lib/current/__init__.py +0 -6
- cognite/neat/_workflows/steps/lib/current/graph_extractor.py +0 -100
- cognite/neat/_workflows/steps/lib/current/graph_loader.py +0 -51
- cognite/neat/_workflows/steps/lib/current/graph_store.py +0 -48
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +0 -537
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +0 -398
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +0 -106
- cognite/neat/_workflows/steps/lib/io/__init__.py +0 -1
- cognite/neat/_workflows/steps/lib/io/io_steps.py +0 -393
- cognite/neat/_workflows/steps/step_model.py +0 -79
- cognite/neat/_workflows/steps_registry.py +0 -218
- cognite/neat/_workflows/tasks.py +0 -18
- cognite/neat/_workflows/triggers.py +0 -169
- cognite/neat/_workflows/utils.py +0 -19
- cognite_neat-0.103.1.dist-info/RECORD +0 -275
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.103.1.dist-info → cognite_neat-0.105.0.dist-info}/entry_points.txt +0 -0
|
@@ -5,17 +5,18 @@ from urllib.parse import quote
|
|
|
5
5
|
|
|
6
6
|
import rdflib
|
|
7
7
|
from rdflib import RDF, XSD, Graph, Namespace, URIRef
|
|
8
|
+
from rdflib.query import ResultRow
|
|
8
9
|
|
|
9
10
|
from cognite.neat._constants import UNKNOWN_TYPE
|
|
10
11
|
from cognite.neat._graph.queries import Queries
|
|
11
|
-
from cognite.neat._issues.warnings import
|
|
12
|
+
from cognite.neat._issues.warnings import PropertyDataTypeConversionWarning
|
|
12
13
|
from cognite.neat._utils.auxiliary import string_to_ideal_type
|
|
13
|
-
from cognite.neat._utils.collection_ import iterate_progress_bar
|
|
14
14
|
from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
|
|
15
15
|
|
|
16
|
-
from ._base import BaseTransformer
|
|
16
|
+
from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
|
|
17
17
|
|
|
18
18
|
|
|
19
|
+
# TODO: Standardise
|
|
19
20
|
class SplitMultiValueProperty(BaseTransformer):
|
|
20
21
|
description: str = (
|
|
21
22
|
"SplitMultiValueProperty is a transformer that splits a "
|
|
@@ -75,34 +76,11 @@ class SplitMultiValueProperty(BaseTransformer):
|
|
|
75
76
|
graph.add((s, new_property, o))
|
|
76
77
|
|
|
77
78
|
|
|
78
|
-
class ConvertLiteral(
|
|
79
|
+
class ConvertLiteral(BaseTransformerStandardised):
|
|
79
80
|
description: str = "ConvertLiteral is a transformer that improve data typing of a literal value."
|
|
80
81
|
_use_only_once: bool = False
|
|
81
82
|
_need_changes = frozenset({})
|
|
82
83
|
|
|
83
|
-
_count_by_properties = """SELECT (COUNT(?value) AS ?valueCount)
|
|
84
|
-
WHERE {{
|
|
85
|
-
?instance a <{subject_type}> .
|
|
86
|
-
?instance <{subject_predicate}> ?value
|
|
87
|
-
FILTER(isLiteral(?value))
|
|
88
|
-
}}"""
|
|
89
|
-
|
|
90
|
-
_count_by_properties_uri = """SELECT (COUNT(?value) AS ?valueCount)
|
|
91
|
-
WHERE {{
|
|
92
|
-
?instance a <{subject_type}> .
|
|
93
|
-
?instance <{subject_predicate}> ?value
|
|
94
|
-
FILTER(isIRI(?value))
|
|
95
|
-
}}"""
|
|
96
|
-
|
|
97
|
-
_properties = """SELECT ?instance ?value
|
|
98
|
-
WHERE {{
|
|
99
|
-
?instance a <{subject_type}> .
|
|
100
|
-
?instance <{subject_predicate}> ?value
|
|
101
|
-
|
|
102
|
-
FILTER(isLiteral(?value))
|
|
103
|
-
|
|
104
|
-
}}"""
|
|
105
|
-
|
|
106
84
|
def __init__(
|
|
107
85
|
self,
|
|
108
86
|
subject_type: URIRef,
|
|
@@ -115,89 +93,55 @@ class ConvertLiteral(BaseTransformer):
|
|
|
115
93
|
self._type_name = remove_namespace_from_uri(subject_type)
|
|
116
94
|
self._property_name = remove_namespace_from_uri(subject_predicate)
|
|
117
95
|
|
|
118
|
-
def
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
96
|
+
def _skip_count_query(self) -> str:
|
|
97
|
+
query = """SELECT (COUNT(?value) AS ?valueCount)
|
|
98
|
+
WHERE {{
|
|
99
|
+
?instance a <{subject_type}> .
|
|
100
|
+
?instance <{subject_predicate}> ?value
|
|
101
|
+
FILTER(isIRI(?value))
|
|
102
|
+
}}"""
|
|
103
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
104
|
+
|
|
105
|
+
def _count_query(self) -> str:
|
|
106
|
+
query = """SELECT (COUNT(?value) AS ?valueCount)
|
|
107
|
+
WHERE {{
|
|
108
|
+
?instance a <{subject_type}> .
|
|
109
|
+
?instance <{subject_predicate}> ?value
|
|
110
|
+
FILTER(isLiteral(?value))
|
|
111
|
+
}}"""
|
|
112
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
113
|
+
|
|
114
|
+
def _iterate_query(self) -> str:
|
|
115
|
+
query = """SELECT ?instance ?value
|
|
116
|
+
WHERE {{
|
|
117
|
+
?instance a <{subject_type}> .
|
|
118
|
+
?instance <{subject_predicate}> ?value
|
|
119
|
+
FILTER(isLiteral(?value))
|
|
120
|
+
}}"""
|
|
121
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
122
|
+
|
|
123
|
+
def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
|
|
124
|
+
row_output = RowTransformationOutput()
|
|
125
|
+
|
|
126
|
+
instance, literal = query_result_row
|
|
127
|
+
value = cast(rdflib.Literal, literal).toPython()
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
converted_value = self.conversion(value)
|
|
131
|
+
except Exception as e:
|
|
126
132
|
warnings.warn(
|
|
127
|
-
|
|
128
|
-
f"Skipping {connection_count} of {self._type_name}.{self._property_name} "
|
|
129
|
-
f"as these are connections and not data values."
|
|
130
|
-
),
|
|
133
|
+
PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
|
|
131
134
|
stacklevel=2,
|
|
132
135
|
)
|
|
136
|
+
row_output.add_triples.append((instance, self.subject_predicate, rdflib.Literal(converted_value))) # type: ignore[arg-type]
|
|
137
|
+
row_output.remove_triples.append((instance, self.subject_predicate, literal)) # type: ignore[arg-type]
|
|
138
|
+
row_output.instances_modified_count += 1
|
|
133
139
|
|
|
134
|
-
|
|
135
|
-
subject_type=self.subject_type, subject_predicate=self.subject_predicate
|
|
136
|
-
)
|
|
137
|
-
|
|
138
|
-
property_count_res = list(graph.query(count_query))
|
|
139
|
-
property_count = int(property_count_res[0][0]) # type: ignore [index, arg-type]
|
|
140
|
-
iterate_query = self._properties.format(
|
|
141
|
-
subject_type=self.subject_type, subject_predicate=self.subject_predicate
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
for instance, literal in iterate_progress_bar( # type: ignore[misc]
|
|
145
|
-
graph.query(iterate_query),
|
|
146
|
-
total=property_count,
|
|
147
|
-
description=f"Converting {self._type_name}.{self._property_name}.",
|
|
148
|
-
):
|
|
149
|
-
value = cast(rdflib.Literal, literal).toPython()
|
|
150
|
-
try:
|
|
151
|
-
converted_value = self.conversion(value)
|
|
152
|
-
except Exception as e:
|
|
153
|
-
warnings.warn(
|
|
154
|
-
PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
|
|
155
|
-
stacklevel=2,
|
|
156
|
-
)
|
|
157
|
-
continue
|
|
158
|
-
|
|
159
|
-
graph.add((instance, self.subject_predicate, rdflib.Literal(converted_value)))
|
|
160
|
-
graph.remove((instance, self.subject_predicate, literal))
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
class LiteralToEntity(BaseTransformer):
|
|
164
|
-
description = "Converts a literal value to new entity"
|
|
140
|
+
return row_output
|
|
165
141
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
?instance <{subject_predicate}> ?property
|
|
170
|
-
FILTER(isLiteral(?property))
|
|
171
|
-
}}"""
|
|
172
|
-
_count_connections_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
173
|
-
WHERE {{
|
|
174
|
-
?instance a <{subject_type}> .
|
|
175
|
-
?instance <{subject_predicate}> ?property
|
|
176
|
-
FILTER(isIRI(?property))
|
|
177
|
-
}}"""
|
|
178
|
-
|
|
179
|
-
_properties_of_type = """SELECT ?instance ?property
|
|
180
|
-
WHERE {{
|
|
181
|
-
?instance a <{subject_type}> .
|
|
182
|
-
?instance <{subject_predicate}> ?property
|
|
183
|
-
FILTER(isLiteral(?property))
|
|
184
|
-
}}"""
|
|
185
|
-
|
|
186
|
-
_count_properties = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
187
|
-
WHERE {{
|
|
188
|
-
?instance <{subject_predicate}> ?property
|
|
189
|
-
FILTER(isLiteral(?property))
|
|
190
|
-
}}"""
|
|
191
|
-
_count_connections = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
192
|
-
WHERE {{
|
|
193
|
-
?instance <{subject_predicate}> ?property
|
|
194
|
-
FILTER(isIRI(?property))
|
|
195
|
-
}}"""
|
|
196
|
-
_properties = """SELECT ?instance ?property
|
|
197
|
-
WHERE {{
|
|
198
|
-
?instance <{subject_predicate}> ?property
|
|
199
|
-
FILTER(isLiteral(?property))
|
|
200
|
-
}}"""
|
|
142
|
+
|
|
143
|
+
class LiteralToEntity(BaseTransformerStandardised):
|
|
144
|
+
description = "Converts a literal value to new entity"
|
|
201
145
|
|
|
202
146
|
def __init__(
|
|
203
147
|
self, subject_type: URIRef | None, subject_predicate: URIRef, entity_type: str, new_property: str | None = None
|
|
@@ -207,54 +151,147 @@ class LiteralToEntity(BaseTransformer):
|
|
|
207
151
|
self.entity_type = entity_type
|
|
208
152
|
self.new_property = new_property
|
|
209
153
|
|
|
210
|
-
def
|
|
154
|
+
def _iterate_query(self) -> str:
|
|
211
155
|
if self.subject_type is None:
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
156
|
+
query = """SELECT ?instance ?property
|
|
157
|
+
WHERE {{
|
|
158
|
+
?instance <{subject_predicate}> ?property
|
|
159
|
+
FILTER(isLiteral(?property))
|
|
160
|
+
}}"""
|
|
161
|
+
return query.format(subject_predicate=self.subject_predicate)
|
|
215
162
|
else:
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
163
|
+
query = """SELECT ?instance ?property
|
|
164
|
+
WHERE {{
|
|
165
|
+
?instance a <{subject_type}> .
|
|
166
|
+
?instance <{subject_predicate}> ?property
|
|
167
|
+
FILTER(isLiteral(?property))
|
|
168
|
+
}}"""
|
|
169
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
170
|
+
|
|
171
|
+
def _skip_count_query(self) -> str:
|
|
172
|
+
if self.subject_type is None:
|
|
173
|
+
query = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
174
|
+
WHERE {{
|
|
175
|
+
?instance <{subject_predicate}> ?property
|
|
176
|
+
FILTER(isIRI(?property))
|
|
177
|
+
}}"""
|
|
178
|
+
return query.format(subject_predicate=self.subject_predicate)
|
|
179
|
+
else:
|
|
180
|
+
query = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
181
|
+
WHERE {{
|
|
182
|
+
?instance a <{subject_type}> .
|
|
183
|
+
?instance <{subject_predicate}> ?property
|
|
184
|
+
FILTER(isIRI(?property))
|
|
185
|
+
}}"""
|
|
186
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
187
|
+
|
|
188
|
+
def _count_query(self) -> str:
|
|
189
|
+
if self.subject_type is None:
|
|
190
|
+
query = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
191
|
+
WHERE {{
|
|
192
|
+
?instance <{subject_predicate}> ?property
|
|
193
|
+
FILTER(isLiteral(?property))
|
|
194
|
+
}}"""
|
|
195
|
+
return query.format(subject_predicate=self.subject_predicate)
|
|
196
|
+
else:
|
|
197
|
+
query = """SELECT (COUNT(?property) AS ?propertyCount)
|
|
198
|
+
WHERE {{
|
|
199
|
+
?instance a <{subject_type}> .
|
|
200
|
+
?instance <{subject_predicate}> ?property
|
|
201
|
+
FILTER(isLiteral(?property))
|
|
202
|
+
}}"""
|
|
225
203
|
|
|
226
|
-
|
|
227
|
-
connection_count = int(connection_count_res[0][0]) # type: ignore [index, arg-type]
|
|
228
|
-
if connection_count > 0:
|
|
229
|
-
warnings.warn(
|
|
230
|
-
NeatValueWarning(
|
|
231
|
-
f"Skipping {connection_count} of {remove_namespace_from_uri(self.subject_predicate)} "
|
|
232
|
-
f"as these are connections and not data values."
|
|
233
|
-
),
|
|
234
|
-
stacklevel=2,
|
|
235
|
-
)
|
|
204
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
236
205
|
|
|
237
|
-
|
|
238
|
-
|
|
206
|
+
def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
|
|
207
|
+
row_output = RowTransformationOutput()
|
|
239
208
|
|
|
240
|
-
instance
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
209
|
+
instance, literal = query_result_row
|
|
210
|
+
value = cast(rdflib.Literal, literal).toPython()
|
|
211
|
+
namespace = Namespace(get_namespace(instance)) # type: ignore[arg-type]
|
|
212
|
+
entity_type = namespace[self.entity_type]
|
|
213
|
+
new_entity = namespace[f"{self.entity_type}_{quote(value)!s}"]
|
|
214
|
+
row_output.add_triples.append((new_entity, RDF.type, entity_type))
|
|
215
|
+
row_output.instances_added_count += 1 # we add one new entity
|
|
216
|
+
|
|
217
|
+
if self.new_property is not None:
|
|
218
|
+
row_output.add_triples.append((new_entity, namespace[self.new_property], rdflib.Literal(value))) # type: ignore[arg-type]
|
|
219
|
+
row_output.instances_modified_count += 1 # we modify the new entity
|
|
220
|
+
|
|
221
|
+
row_output.add_triples.append((instance, self.subject_predicate, new_entity)) # type: ignore[arg-type]
|
|
222
|
+
row_output.remove_triples.append((instance, self.subject_predicate, literal)) # type: ignore[arg-type]
|
|
223
|
+
row_output.instances_modified_count += 1 # we modify the old entity
|
|
224
|
+
|
|
225
|
+
return row_output
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class ConnectionToLiteral(BaseTransformerStandardised):
|
|
229
|
+
description = "Converts an entity connection to a literal value"
|
|
230
|
+
|
|
231
|
+
def __init__(self, subject_type: URIRef | None, subject_predicate: URIRef) -> None:
|
|
232
|
+
self.subject_type = subject_type
|
|
233
|
+
self.subject_predicate = subject_predicate
|
|
234
|
+
|
|
235
|
+
def _iterate_query(self) -> str:
|
|
236
|
+
if self.subject_type is None:
|
|
237
|
+
query = """SELECT ?instance ?object
|
|
238
|
+
WHERE {{
|
|
239
|
+
?instance <{subject_predicate}> ?object
|
|
240
|
+
FILTER(isIRI(?object))
|
|
241
|
+
}}"""
|
|
242
|
+
return query.format(subject_predicate=self.subject_predicate)
|
|
243
|
+
else:
|
|
244
|
+
query = """SELECT ?instance ?object
|
|
245
|
+
WHERE {{
|
|
246
|
+
?instance a <{subject_type}> .
|
|
247
|
+
?instance <{subject_predicate}> ?object
|
|
248
|
+
FILTER(isIRI(?object))
|
|
249
|
+
}}"""
|
|
250
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
251
|
+
|
|
252
|
+
def _skip_count_query(self) -> str:
|
|
253
|
+
if self.subject_type is None:
|
|
254
|
+
query = """SELECT (COUNT(?object) AS ?objectCount)
|
|
255
|
+
WHERE {{
|
|
256
|
+
?instance <{subject_predicate}> ?object
|
|
257
|
+
FILTER(isLiteral(?object))
|
|
258
|
+
}}"""
|
|
259
|
+
return query.format(subject_predicate=self.subject_predicate)
|
|
260
|
+
else:
|
|
261
|
+
query = """SELECT (COUNT(?object) AS ?objectCount)
|
|
262
|
+
WHERE {{
|
|
263
|
+
?instance a <{subject_type}> .
|
|
264
|
+
?instance <{subject_predicate}> ?object
|
|
265
|
+
FILTER(isLiteral(?object))
|
|
266
|
+
}}"""
|
|
267
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
268
|
+
|
|
269
|
+
def _count_query(self) -> str:
|
|
270
|
+
if self.subject_type is None:
|
|
271
|
+
query = """SELECT (COUNT(?object) AS ?objectCount)
|
|
272
|
+
WHERE {{
|
|
273
|
+
?instance <{subject_predicate}> ?object
|
|
274
|
+
FILTER(isIRI(?object))
|
|
275
|
+
}}"""
|
|
276
|
+
return query.format(subject_predicate=self.subject_predicate)
|
|
277
|
+
else:
|
|
278
|
+
query = """SELECT (COUNT(?object) AS ?objectCount)
|
|
279
|
+
WHERE {{
|
|
280
|
+
?instance a <{subject_type}> .
|
|
281
|
+
?instance <{subject_predicate}> ?object
|
|
282
|
+
FILTER(isIRI(?object))
|
|
283
|
+
}}"""
|
|
284
|
+
|
|
285
|
+
return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
|
|
286
|
+
|
|
287
|
+
def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
|
|
288
|
+
row_output = RowTransformationOutput()
|
|
289
|
+
|
|
290
|
+
instance, object_entity = cast(tuple[URIRef, URIRef], query_result_row)
|
|
291
|
+
value = remove_namespace_from_uri(object_entity)
|
|
292
|
+
|
|
293
|
+
row_output.add_triples.append((instance, self.subject_predicate, rdflib.Literal(value)))
|
|
294
|
+
row_output.remove_triples.append((instance, self.subject_predicate, object_entity))
|
|
295
|
+
row_output.instances_modified_count += 1
|
|
296
|
+
|
|
297
|
+
return row_output
|
cognite/neat/_issues/__init__.py
CHANGED
|
@@ -3,7 +3,6 @@ as some helper classes to handle them like NeatIssueList"""
|
|
|
3
3
|
|
|
4
4
|
from ._base import (
|
|
5
5
|
DefaultWarning,
|
|
6
|
-
FutureResult,
|
|
7
6
|
IssueList,
|
|
8
7
|
MultiValueError,
|
|
9
8
|
NeatError,
|
|
@@ -16,7 +15,6 @@ from ._base import (
|
|
|
16
15
|
|
|
17
16
|
__all__ = [
|
|
18
17
|
"DefaultWarning",
|
|
19
|
-
"FutureResult",
|
|
20
18
|
"IssueList",
|
|
21
19
|
"MultiValueError",
|
|
22
20
|
"NeatError",
|
cognite/neat/_issues/_base.py
CHANGED
|
@@ -390,11 +390,16 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
|
|
|
390
390
|
"""This is a generic list of NeatIssues."""
|
|
391
391
|
|
|
392
392
|
def __init__(
|
|
393
|
-
self,
|
|
393
|
+
self,
|
|
394
|
+
issues: Sequence[T_NeatIssue] | None = None,
|
|
395
|
+
title: str | None = None,
|
|
396
|
+
action: str | None = None,
|
|
397
|
+
hint: str | None = None,
|
|
394
398
|
):
|
|
395
399
|
super().__init__(issues or [])
|
|
396
400
|
self.title = title
|
|
397
401
|
self.action = action
|
|
402
|
+
self.hint = hint
|
|
398
403
|
|
|
399
404
|
@property
|
|
400
405
|
def errors(self) -> Self:
|
|
@@ -406,6 +411,11 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
|
|
|
406
411
|
"""Return True if this list contains any errors."""
|
|
407
412
|
return any(isinstance(issue, NeatError) for issue in self)
|
|
408
413
|
|
|
414
|
+
@property
|
|
415
|
+
def has_warnings(self) -> bool:
|
|
416
|
+
"""Return True if this list contains any warnings."""
|
|
417
|
+
return any(isinstance(issue, NeatWarning) for issue in self)
|
|
418
|
+
|
|
409
419
|
@property
|
|
410
420
|
def warnings(self) -> Self:
|
|
411
421
|
"""Return all the warnings in this list."""
|
|
@@ -415,6 +425,10 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
|
|
|
415
425
|
"""Return True if this list contains any errors of the given type."""
|
|
416
426
|
return any(isinstance(issue, error_type) for issue in self)
|
|
417
427
|
|
|
428
|
+
def has_warning_type(self, warning_type: type[NeatWarning]) -> bool:
|
|
429
|
+
"""Return True if this list contains any warnings of the given type."""
|
|
430
|
+
return any(isinstance(issue, warning_type) for issue in self)
|
|
431
|
+
|
|
418
432
|
def as_errors(self, operation: str = "Operation failed") -> ExceptionGroup:
|
|
419
433
|
"""Return an ExceptionGroup with all the errors in this list."""
|
|
420
434
|
return ExceptionGroup(
|
|
@@ -455,14 +469,31 @@ class IssueList(NeatIssueList[NeatIssue]):
|
|
|
455
469
|
"""This is a list of NeatIssues."""
|
|
456
470
|
|
|
457
471
|
def _repr_html_(self) -> str | None:
|
|
458
|
-
if not self:
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
472
|
+
if self.action and not self:
|
|
473
|
+
header = f"Success: {self.action}"
|
|
474
|
+
elif self.action and self.has_errors:
|
|
475
|
+
header = f"Failed: {self.action}"
|
|
476
|
+
elif self.action and self.has_warnings:
|
|
477
|
+
header = f"Succeeded with warnings: {self.action}"
|
|
478
|
+
elif not self:
|
|
479
|
+
header = "No issues found"
|
|
464
480
|
else:
|
|
465
|
-
|
|
481
|
+
header = ""
|
|
482
|
+
|
|
483
|
+
body = f"<p>{header}</p>"
|
|
484
|
+
|
|
485
|
+
if self:
|
|
486
|
+
df = self.to_pandas()
|
|
487
|
+
agg_df = df["NeatIssue"].value_counts().to_frame()
|
|
488
|
+
if len(agg_df) < 10:
|
|
489
|
+
table = agg_df._repr_html_() # type: ignore[operator]
|
|
490
|
+
else:
|
|
491
|
+
table = agg_df.head()._repr_html_() # type: ignore[operator]
|
|
492
|
+
body += f"<br />{table}"
|
|
493
|
+
|
|
494
|
+
if self.hint:
|
|
495
|
+
body += f"<br />Hint: {self.hint}"
|
|
496
|
+
return body
|
|
466
497
|
|
|
467
498
|
|
|
468
499
|
T_Cls = TypeVar("T_Cls")
|
|
@@ -478,55 +509,35 @@ def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[t
|
|
|
478
509
|
|
|
479
510
|
|
|
480
511
|
@contextmanager
|
|
481
|
-
def catch_warnings(
|
|
482
|
-
issues: IssueList | None = None,
|
|
483
|
-
warning_cls: type[NeatWarning] = DefaultWarning,
|
|
484
|
-
) -> Iterator[None]:
|
|
512
|
+
def catch_warnings() -> Iterator[IssueList]:
|
|
485
513
|
"""Catch warnings and append them to the issues list."""
|
|
514
|
+
issues = IssueList()
|
|
486
515
|
with warnings.catch_warnings(record=True) as warning_logger:
|
|
487
516
|
warnings.simplefilter("always")
|
|
488
517
|
try:
|
|
489
|
-
yield
|
|
518
|
+
yield issues
|
|
490
519
|
finally:
|
|
491
|
-
if warning_logger
|
|
492
|
-
issues.extend([
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
class FutureResult:
|
|
496
|
-
def __init__(self) -> None:
|
|
497
|
-
self._result: Literal["success", "failure", "pending"] = "pending"
|
|
498
|
-
|
|
499
|
-
@property
|
|
500
|
-
def result(self) -> Literal["success", "failure", "pending"]:
|
|
501
|
-
return self._result
|
|
520
|
+
if warning_logger:
|
|
521
|
+
issues.extend([NeatWarning.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
|
|
502
522
|
|
|
503
523
|
|
|
504
524
|
@contextmanager
|
|
505
|
-
def catch_issues(
|
|
506
|
-
issues: IssueList,
|
|
507
|
-
error_cls: type[NeatError] = NeatError,
|
|
508
|
-
warning_cls: type[NeatWarning] = NeatWarning,
|
|
509
|
-
error_args: dict[str, Any] | None = None,
|
|
510
|
-
) -> Iterator[FutureResult]:
|
|
525
|
+
def catch_issues(error_args: dict[str, Any] | None = None) -> Iterator[IssueList]:
|
|
511
526
|
"""This is an internal help function to handle issues and warnings.
|
|
512
527
|
|
|
513
528
|
Args:
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
529
|
+
error_args: Additional arguments to pass to the error class. The only use case as of (2025-01-03) is to pass
|
|
530
|
+
the read_info_by_sheet to the error class such that the row numbers can be adjusted to match the source
|
|
531
|
+
spreadsheet.
|
|
517
532
|
|
|
518
533
|
Returns:
|
|
519
|
-
|
|
534
|
+
IssueList: The list of issues.
|
|
535
|
+
|
|
520
536
|
"""
|
|
521
|
-
with catch_warnings(issues
|
|
522
|
-
future_result = FutureResult()
|
|
537
|
+
with catch_warnings() as issues:
|
|
523
538
|
try:
|
|
524
|
-
yield
|
|
539
|
+
yield issues
|
|
525
540
|
except ValidationError as e:
|
|
526
|
-
issues.extend(
|
|
527
|
-
future_result._result = "failure"
|
|
541
|
+
issues.extend(NeatError.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
|
|
528
542
|
except (NeatError, MultiValueError) as e:
|
|
529
|
-
issues.extend(
|
|
530
|
-
future_result._result = "failure"
|
|
531
|
-
else:
|
|
532
|
-
future_result._result = "success"
|
|
543
|
+
issues.extend(NeatError.from_errors([e], **(error_args or {}))) # type: ignore[arg-type, list-item]
|
|
@@ -13,7 +13,7 @@ from ._external import (
|
|
|
13
13
|
FileReadWarning,
|
|
14
14
|
FileTypeUnexpectedWarning,
|
|
15
15
|
)
|
|
16
|
-
from ._general import NeatValueWarning, NotSupportedWarning, RegexViolationWarning
|
|
16
|
+
from ._general import MissingCogniteClientWarning, NeatValueWarning, NotSupportedWarning, RegexViolationWarning
|
|
17
17
|
from ._models import (
|
|
18
18
|
BreakingModelingPrincipleWarning,
|
|
19
19
|
CDFNotSupportedWarning,
|
|
@@ -41,6 +41,7 @@ from ._resources import (
|
|
|
41
41
|
ResourceRetrievalWarning,
|
|
42
42
|
ResourcesDuplicatedWarning,
|
|
43
43
|
ResourceTypeNotSupportedWarning,
|
|
44
|
+
ResourceUnknownWarning,
|
|
44
45
|
)
|
|
45
46
|
|
|
46
47
|
__all__ = [
|
|
@@ -53,6 +54,7 @@ __all__ = [
|
|
|
53
54
|
"FileMissingRequiredFieldWarning",
|
|
54
55
|
"FileReadWarning",
|
|
55
56
|
"FileTypeUnexpectedWarning",
|
|
57
|
+
"MissingCogniteClientWarning",
|
|
56
58
|
"NeatValueWarning",
|
|
57
59
|
"NotSupportedHasDataFilterLimitWarning",
|
|
58
60
|
"NotSupportedViewContainerLimitWarning",
|
|
@@ -73,6 +75,7 @@ __all__ = [
|
|
|
73
75
|
"ResourceRegexViolationWarning",
|
|
74
76
|
"ResourceRetrievalWarning",
|
|
75
77
|
"ResourceTypeNotSupportedWarning",
|
|
78
|
+
"ResourceUnknownWarning",
|
|
76
79
|
"ResourcesDuplicatedWarning",
|
|
77
80
|
"UndefinedViewWarning",
|
|
78
81
|
"UserModelingWarning",
|
|
@@ -27,3 +27,10 @@ class RegexViolationWarning(NeatWarning):
|
|
|
27
27
|
identifier: str
|
|
28
28
|
pattern_name: str
|
|
29
29
|
motivation: str | None = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(unsafe_hash=True)
|
|
33
|
+
class MissingCogniteClientWarning(NeatWarning):
|
|
34
|
+
"""Missing Cognite Client required for {functionality}"""
|
|
35
|
+
|
|
36
|
+
functionality: str
|
|
@@ -21,7 +21,7 @@ class ResourceRegexViolationWarning(ResourceNeatWarning):
|
|
|
21
21
|
|
|
22
22
|
fix = (
|
|
23
23
|
"Either export the data model and make the necessary changes manually"
|
|
24
|
-
" or run prepare.cdf_compliant_external_ids."
|
|
24
|
+
" or run prepare.data_model.cdf_compliant_external_ids."
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
location: str
|
|
@@ -39,6 +39,17 @@ class ResourceNotFoundWarning(ResourceNeatWarning, Generic[T_Identifier, T_Refer
|
|
|
39
39
|
referred_type: str
|
|
40
40
|
|
|
41
41
|
|
|
42
|
+
@dataclass(unsafe_hash=True)
|
|
43
|
+
class ResourceUnknownWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
|
|
44
|
+
"""The {resource_type} with identifier {identifier} referred by {referred_type} {referred_by} is unknown.
|
|
45
|
+
Will continue, but the model is incomplete."""
|
|
46
|
+
|
|
47
|
+
referred_by: T_ReferenceIdentifier
|
|
48
|
+
referred_type: str
|
|
49
|
+
|
|
50
|
+
fix = "You can maybe retrieve the resource from the CDF."
|
|
51
|
+
|
|
52
|
+
|
|
42
53
|
@dataclass(unsafe_hash=True)
|
|
43
54
|
class ResourceNotDefinedWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
|
|
44
55
|
"""The {resource_type} {identifier} is not defined in the {location}"""
|