cognite-neat 0.109.3__py3-none-any.whl → 0.110.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +2 -0
- cognite/neat/_client/_api/schema.py +17 -1
- cognite/neat/_client/data_classes/schema.py +3 -3
- cognite/neat/_constants.py +11 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +9 -10
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/loaders/_rdf2dms.py +285 -346
- cognite/neat/_graph/queries/_base.py +28 -92
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +61 -0
- cognite/neat/_issues/errors/__init__.py +18 -4
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_rules/_constants.py +9 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +450 -258
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +2 -8
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +59 -25
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/dms/_rules.py +3 -1
- cognite/neat/_rules/models/dms/_rules_input.py +4 -0
- cognite/neat/_rules/models/dms/_validation.py +14 -4
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +2 -1
- cognite/neat/_rules/models/information/_validation.py +3 -1
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +242 -43
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +4 -4
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +21 -17
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +0 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +63 -12
- cognite/neat/_store/_graph_store.py +5 -246
- cognite/neat/_utils/rdf_.py +2 -2
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +51 -32
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/RECORD +62 -64
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import warnings
|
|
2
1
|
from collections import defaultdict
|
|
3
2
|
from collections.abc import Iterable
|
|
4
3
|
from typing import Literal, cast, overload
|
|
@@ -9,14 +8,9 @@ from rdflib.graph import DATASET_DEFAULT_GRAPH_ID
|
|
|
9
8
|
from rdflib.query import ResultRow
|
|
10
9
|
|
|
11
10
|
from cognite.neat._constants import NEAT
|
|
12
|
-
from cognite.neat._rules._constants import EntityTypes
|
|
13
|
-
from cognite.neat._rules.models.entities import ClassEntity
|
|
14
|
-
from cognite.neat._rules.models.information import InformationRules
|
|
15
11
|
from cognite.neat._shared import InstanceType
|
|
16
12
|
from cognite.neat._utils.rdf_ import remove_instance_ids_in_batch, remove_namespace_from_uri
|
|
17
13
|
|
|
18
|
-
from ._construct import build_construct_query
|
|
19
|
-
|
|
20
14
|
|
|
21
15
|
class Queries:
|
|
22
16
|
"""Helper class for storing standard queries for the graph store."""
|
|
@@ -24,11 +18,9 @@ class Queries:
|
|
|
24
18
|
def __init__(
|
|
25
19
|
self,
|
|
26
20
|
dataset: Dataset,
|
|
27
|
-
rules: dict[URIRef, InformationRules] | None = None,
|
|
28
21
|
default_named_graph: URIRef | None = None,
|
|
29
22
|
):
|
|
30
23
|
self.dataset = dataset
|
|
31
|
-
self.rules = rules or {}
|
|
32
24
|
self.default_named_graph = default_named_graph or DATASET_DEFAULT_GRAPH_ID
|
|
33
25
|
|
|
34
26
|
def graph(self, named_graph: URIRef | None = None) -> Graph:
|
|
@@ -126,38 +118,6 @@ class Queries:
|
|
|
126
118
|
# Select queries gives an iterable of result rows
|
|
127
119
|
return cast(list[ResultRow], list(self.graph(named_graph).query(query)))
|
|
128
120
|
|
|
129
|
-
def triples_of_type_instances(
|
|
130
|
-
self, rdf_type: str | URIRef, named_graph: URIRef | None = None
|
|
131
|
-
) -> list[tuple[str, str, str]]:
|
|
132
|
-
"""Get all triples of a given type.
|
|
133
|
-
|
|
134
|
-
Args:
|
|
135
|
-
rdf_type: Type URI to query
|
|
136
|
-
named_graph: Named graph to query over, default None (default graph)
|
|
137
|
-
"""
|
|
138
|
-
named_graph = named_graph or self.default_named_graph
|
|
139
|
-
if isinstance(rdf_type, URIRef):
|
|
140
|
-
rdf_uri = rdf_type
|
|
141
|
-
elif isinstance(rdf_type, str) and self.rules and self.rules.get(named_graph):
|
|
142
|
-
rdf_uri = self.rules[named_graph].metadata.namespace[rdf_type]
|
|
143
|
-
else:
|
|
144
|
-
warnings.warn(
|
|
145
|
-
"Unknown namespace. Please either provide a URIRef or set the rules of the store.",
|
|
146
|
-
stacklevel=2,
|
|
147
|
-
)
|
|
148
|
-
return []
|
|
149
|
-
|
|
150
|
-
query = (
|
|
151
|
-
"SELECT ?instance ?prop ?value "
|
|
152
|
-
f"WHERE {{ ?instance a <{rdf_uri}> . ?instance ?prop ?value . }} "
|
|
153
|
-
"order by ?instance"
|
|
154
|
-
)
|
|
155
|
-
|
|
156
|
-
result = self.graph(named_graph).query(query)
|
|
157
|
-
|
|
158
|
-
# We cannot include the RDF.type in case there is a neat:type property
|
|
159
|
-
return [remove_namespace_from_uri(list(triple)) for triple in result if triple[1] != RDF.type] # type: ignore[misc, index, arg-type]
|
|
160
|
-
|
|
161
121
|
def type_with_property(self, type_: URIRef, property_uri: URIRef, named_graph: URIRef | None = None) -> bool:
|
|
162
122
|
"""Check if a property exists in the graph store
|
|
163
123
|
|
|
@@ -205,9 +165,8 @@ class Queries:
|
|
|
205
165
|
def describe(
|
|
206
166
|
self,
|
|
207
167
|
instance_id: URIRef,
|
|
208
|
-
instance_type:
|
|
168
|
+
instance_type: URIRef | None = None,
|
|
209
169
|
property_renaming_config: dict | None = None,
|
|
210
|
-
property_types: dict[str, EntityTypes] | None = None,
|
|
211
170
|
named_graph: URIRef | None = None,
|
|
212
171
|
) -> tuple[str, dict[str | InstanceType, list[str]]] | None:
|
|
213
172
|
"""DESCRIBE instance for a given class from the graph store
|
|
@@ -216,7 +175,6 @@ class Queries:
|
|
|
216
175
|
instance_id: Instance id for which we want to generate query
|
|
217
176
|
instance_type: Type of the instance, default None (will be inferred from triples)
|
|
218
177
|
property_renaming_config: Dictionary to rename properties, default None (no renaming)
|
|
219
|
-
property_types: Dictionary of property types, default None (helper for removal of namespace)
|
|
220
178
|
named_graph: Named graph to query over, default None (default graph)
|
|
221
179
|
|
|
222
180
|
|
|
@@ -261,7 +219,9 @@ class Queries:
|
|
|
261
219
|
else:
|
|
262
220
|
# guarding against multiple rdf:type values as this is not allowed in CDF
|
|
263
221
|
if RDF.type not in property_values:
|
|
264
|
-
property_values[RDF.type].append(
|
|
222
|
+
property_values[RDF.type].append(
|
|
223
|
+
remove_namespace_from_uri(instance_type, validation="prefix") if instance_type else value
|
|
224
|
+
)
|
|
265
225
|
else:
|
|
266
226
|
# we should not have multiple rdf:type values
|
|
267
227
|
continue
|
|
@@ -273,52 +233,6 @@ class Queries:
|
|
|
273
233
|
else:
|
|
274
234
|
return None
|
|
275
235
|
|
|
276
|
-
def construct_instances_of_class(
|
|
277
|
-
self,
|
|
278
|
-
class_: str,
|
|
279
|
-
properties_optional: bool = True,
|
|
280
|
-
instance_id: URIRef | None = None,
|
|
281
|
-
named_graph: URIRef | None = None,
|
|
282
|
-
) -> list[tuple[str, str, str]]:
|
|
283
|
-
"""CONSTRUCT instances for a given class from the graph store
|
|
284
|
-
|
|
285
|
-
Args:
|
|
286
|
-
class_: Class entity for which we want to generate query
|
|
287
|
-
properties_optional: Whether to make all properties optional, default True
|
|
288
|
-
instance_ids: List of instance ids to filter on, default None (all)
|
|
289
|
-
named_graph: Named graph to query over, default None (default graph
|
|
290
|
-
|
|
291
|
-
Returns:
|
|
292
|
-
List of triples for instances of the given class
|
|
293
|
-
"""
|
|
294
|
-
named_graph = named_graph or self.default_named_graph
|
|
295
|
-
if (
|
|
296
|
-
self.rules
|
|
297
|
-
and self.rules.get(named_graph)
|
|
298
|
-
and (
|
|
299
|
-
query := build_construct_query(
|
|
300
|
-
class_=ClassEntity(
|
|
301
|
-
prefix=self.rules[named_graph].metadata.prefix,
|
|
302
|
-
suffix=class_,
|
|
303
|
-
),
|
|
304
|
-
graph=self.graph(named_graph),
|
|
305
|
-
rules=self.rules[named_graph],
|
|
306
|
-
properties_optional=properties_optional,
|
|
307
|
-
instance_id=instance_id,
|
|
308
|
-
)
|
|
309
|
-
)
|
|
310
|
-
):
|
|
311
|
-
result = self.graph(named_graph).query(query)
|
|
312
|
-
|
|
313
|
-
# We cannot include the RDF.type in case there is a neat:type property
|
|
314
|
-
return [remove_namespace_from_uri(cast(ResultRow, triple)) for triple in result if triple[1] != RDF.type] # type: ignore[misc, index, arg-type]
|
|
315
|
-
else:
|
|
316
|
-
warnings.warn(
|
|
317
|
-
"No rules found for the graph store, returning empty list.",
|
|
318
|
-
stacklevel=2,
|
|
319
|
-
)
|
|
320
|
-
return []
|
|
321
|
-
|
|
322
236
|
def list_triples(self, limit: int = 25, named_graph: URIRef | None = None) -> list[ResultRow]:
|
|
323
237
|
"""List triples in the graph store
|
|
324
238
|
|
|
@@ -346,7 +260,7 @@ class Queries:
|
|
|
346
260
|
def list_types(
|
|
347
261
|
self,
|
|
348
262
|
remove_namespace: bool = False,
|
|
349
|
-
limit: int = 25,
|
|
263
|
+
limit: int | None = 25,
|
|
350
264
|
named_graph: URIRef | None = None,
|
|
351
265
|
) -> list[ResultRow] | list[str]:
|
|
352
266
|
"""List types in the graph store
|
|
@@ -358,7 +272,9 @@ class Queries:
|
|
|
358
272
|
Returns:
|
|
359
273
|
List of types
|
|
360
274
|
"""
|
|
361
|
-
query =
|
|
275
|
+
query = "SELECT DISTINCT ?type WHERE { ?subject a ?type }"
|
|
276
|
+
if limit is not None:
|
|
277
|
+
query += f" LIMIT {limit}"
|
|
362
278
|
result = cast(list[ResultRow], list(self.graph(named_graph).query(query)))
|
|
363
279
|
if remove_namespace:
|
|
364
280
|
return [remove_namespace_from_uri(res[0]) for res in result]
|
|
@@ -438,3 +354,23 @@ class Queries:
|
|
|
438
354
|
result[remove_namespace_from_uri(instance)] = remove_namespace_from_uri(types.split(","))
|
|
439
355
|
|
|
440
356
|
return result
|
|
357
|
+
|
|
358
|
+
def count_of_type(self, class_uri: URIRef, named_graph: URIRef | None = None) -> int:
|
|
359
|
+
query = f"SELECT (COUNT(?instance) AS ?instanceCount) WHERE {{ ?instance a <{class_uri}> }}"
|
|
360
|
+
return int(next(iter(self.graph(named_graph).query(query)))[0]) # type: ignore[arg-type, index]
|
|
361
|
+
|
|
362
|
+
def list_instances_ids_by_space(
|
|
363
|
+
self, space_property: URIRef, named_graph: URIRef | None = None
|
|
364
|
+
) -> Iterable[tuple[URIRef, str]]:
|
|
365
|
+
"""Returns instance ids by space"""
|
|
366
|
+
query = f"""SELECT DISTINCT ?instance ?space
|
|
367
|
+
WHERE {{?instance <{space_property}> ?space}}"""
|
|
368
|
+
|
|
369
|
+
for result in cast(Iterable[ResultRow], self.graph(named_graph).query(query)):
|
|
370
|
+
instance_id, space = cast(tuple[URIRef, URIRef | RdfLiteral], result)
|
|
371
|
+
if isinstance(space, URIRef):
|
|
372
|
+
yield instance_id, remove_namespace_from_uri(space)
|
|
373
|
+
elif isinstance(space, RdfLiteral):
|
|
374
|
+
yield instance_id, str(space.toPython())
|
|
375
|
+
else:
|
|
376
|
+
yield instance_id, str(space)
|
|
@@ -16,12 +16,11 @@ from ._prune_graph import (
|
|
|
16
16
|
PruneInstancesOfUnknownType,
|
|
17
17
|
PruneTypes,
|
|
18
18
|
)
|
|
19
|
-
from ._rdfpath import
|
|
19
|
+
from ._rdfpath import MakeConnectionOnExactMatch
|
|
20
20
|
from ._value_type import ConnectionToLiteral, ConvertLiteral, LiteralToEntity, SetType, SplitMultiValueProperty
|
|
21
21
|
|
|
22
22
|
__all__ = [
|
|
23
23
|
"AddAssetDepth",
|
|
24
|
-
"AddSelfReferenceProperty",
|
|
25
24
|
"AssetEventConnector",
|
|
26
25
|
"AssetFileConnector",
|
|
27
26
|
"AssetRelationshipConnector",
|
|
@@ -49,7 +48,6 @@ Transformers = (
|
|
|
49
48
|
| AssetFileConnector
|
|
50
49
|
| AssetEventConnector
|
|
51
50
|
| AssetRelationshipConnector
|
|
52
|
-
| AddSelfReferenceProperty
|
|
53
51
|
| SplitMultiValueProperty
|
|
54
52
|
| RelationshipAsEdgeTransformer
|
|
55
53
|
| MakeConnectionOnExactMatch
|
|
@@ -1,59 +1,12 @@
|
|
|
1
1
|
from typing import cast
|
|
2
2
|
from urllib.parse import quote
|
|
3
3
|
|
|
4
|
-
from rdflib import
|
|
4
|
+
from rdflib import Namespace, URIRef
|
|
5
5
|
from rdflib.query import ResultRow
|
|
6
6
|
|
|
7
|
-
from cognite.neat._rules.analysis import InformationAnalysis
|
|
8
|
-
from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
|
|
9
|
-
from cognite.neat._rules.models.information import InformationRules
|
|
10
7
|
from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
|
|
11
8
|
|
|
12
|
-
from ._base import
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class ReduceHopTraversal(BaseTransformer):
|
|
16
|
-
"""ReduceHopTraversal is a transformer that reduces the number of hops to direct connection."""
|
|
17
|
-
|
|
18
|
-
...
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# TODO: Standardise
|
|
22
|
-
class AddSelfReferenceProperty(BaseTransformer):
|
|
23
|
-
description: str = "Adds property that contains id of reference to all references of given class in Rules"
|
|
24
|
-
_use_only_once: bool = True
|
|
25
|
-
_need_changes = frozenset({})
|
|
26
|
-
_ref_template: str = """SELECT ?s WHERE {{?s a <{type_}>}}"""
|
|
27
|
-
|
|
28
|
-
def __init__(
|
|
29
|
-
self,
|
|
30
|
-
rules: InformationRules,
|
|
31
|
-
):
|
|
32
|
-
self.rules = rules
|
|
33
|
-
self.properties = InformationAnalysis(rules).all_reference_transformations()
|
|
34
|
-
|
|
35
|
-
def transform(self, graph: Graph) -> None:
|
|
36
|
-
for property_ in self.properties:
|
|
37
|
-
prefix = property_.instance_source.traversal.class_.prefix
|
|
38
|
-
suffix = property_.instance_source.traversal.class_.suffix
|
|
39
|
-
|
|
40
|
-
namespace = self.rules.prefixes[prefix] if prefix in self.rules.prefixes else self.rules.metadata.namespace
|
|
41
|
-
|
|
42
|
-
for (reference,) in graph.query(self._ref_template.format(type_=namespace[suffix])): # type: ignore [misc]
|
|
43
|
-
graph.add(
|
|
44
|
-
(
|
|
45
|
-
reference,
|
|
46
|
-
self.rules.metadata.namespace[property_.property_],
|
|
47
|
-
reference,
|
|
48
|
-
)
|
|
49
|
-
)
|
|
50
|
-
|
|
51
|
-
traversal = SingleProperty.from_string(
|
|
52
|
-
class_=property_.view.id,
|
|
53
|
-
property_=f"{self.rules.metadata.prefix}:{property_.property_}",
|
|
54
|
-
)
|
|
55
|
-
|
|
56
|
-
property_.instance_source = RDFPath(traversal=traversal)
|
|
9
|
+
from ._base import BaseTransformerStandardised, RowTransformationOutput
|
|
57
10
|
|
|
58
11
|
|
|
59
12
|
class MakeConnectionOnExactMatch(BaseTransformerStandardised):
|
cognite/neat/_issues/__init__.py
CHANGED
|
@@ -2,24 +2,19 @@
|
|
|
2
2
|
as some helper classes to handle them like NeatIssueList"""
|
|
3
3
|
|
|
4
4
|
from ._base import (
|
|
5
|
-
DefaultWarning,
|
|
6
5
|
IssueList,
|
|
7
6
|
MultiValueError,
|
|
8
7
|
NeatError,
|
|
9
8
|
NeatIssue,
|
|
10
|
-
NeatIssueList,
|
|
11
9
|
NeatWarning,
|
|
12
|
-
catch_issues,
|
|
13
|
-
catch_warnings,
|
|
14
10
|
)
|
|
11
|
+
from ._contextmanagers import catch_issues, catch_warnings
|
|
15
12
|
|
|
16
13
|
__all__ = [
|
|
17
|
-
"DefaultWarning",
|
|
18
14
|
"IssueList",
|
|
19
15
|
"MultiValueError",
|
|
20
16
|
"NeatError",
|
|
21
17
|
"NeatIssue",
|
|
22
|
-
"NeatIssueList",
|
|
23
18
|
"NeatWarning",
|
|
24
19
|
"catch_issues",
|
|
25
20
|
"catch_warnings",
|
cognite/neat/_issues/_base.py
CHANGED
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import sys
|
|
3
3
|
import warnings
|
|
4
|
-
from abc import
|
|
5
|
-
from collections.abc import Collection, Hashable, Iterable, Iterator, Sequence
|
|
6
|
-
from contextlib import contextmanager
|
|
4
|
+
from collections.abc import Collection, Hashable, Iterable, Sequence
|
|
7
5
|
from dataclasses import dataclass, fields
|
|
8
6
|
from functools import total_ordering
|
|
9
7
|
from pathlib import Path
|
|
10
8
|
from types import UnionType
|
|
11
9
|
from typing import Any, ClassVar, Literal, TypeAlias, TypeVar, get_args, get_origin
|
|
12
|
-
from warnings import WarningMessage
|
|
13
10
|
|
|
14
11
|
import pandas as pd
|
|
15
12
|
from cognite.client.data_classes.data_modeling import (
|
|
@@ -18,11 +15,8 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
18
15
|
PropertyId,
|
|
19
16
|
ViewId,
|
|
20
17
|
)
|
|
21
|
-
from pydantic import ValidationError
|
|
22
|
-
from pydantic_core import ErrorDetails
|
|
23
18
|
|
|
24
|
-
from cognite.neat._utils.
|
|
25
|
-
from cognite.neat._utils.text import humanize_collection, to_camel, to_snake
|
|
19
|
+
from cognite.neat._utils.text import humanize_collection, to_camel_case, to_snake_case
|
|
26
20
|
|
|
27
21
|
if sys.version_info < (3, 11):
|
|
28
22
|
from exceptiongroup import ExceptionGroup
|
|
@@ -32,11 +26,10 @@ else:
|
|
|
32
26
|
|
|
33
27
|
|
|
34
28
|
__all__ = [
|
|
35
|
-
"
|
|
29
|
+
"IssueList",
|
|
36
30
|
"MultiValueError",
|
|
37
31
|
"NeatError",
|
|
38
32
|
"NeatIssue",
|
|
39
|
-
"NeatIssueList",
|
|
40
33
|
"NeatWarning",
|
|
41
34
|
]
|
|
42
35
|
|
|
@@ -113,7 +106,7 @@ class NeatIssue:
|
|
|
113
106
|
"""Return a dictionary representation of the issue."""
|
|
114
107
|
variables = vars(self)
|
|
115
108
|
output = {
|
|
116
|
-
|
|
109
|
+
to_camel_case(key): self._dump_value(value)
|
|
117
110
|
for key, value in variables.items()
|
|
118
111
|
if not (value is None or key.startswith("_"))
|
|
119
112
|
}
|
|
@@ -153,7 +146,7 @@ class NeatIssue:
|
|
|
153
146
|
if "NeatIssue" not in data:
|
|
154
147
|
raise NeatValueError("The data does not contain a NeatIssue key.")
|
|
155
148
|
issue_type = data.pop("NeatIssue")
|
|
156
|
-
args = {
|
|
149
|
+
args = {to_snake_case(key): value for key, value in data.items()}
|
|
157
150
|
if issue_type in _NEAT_ERRORS_BY_NAME:
|
|
158
151
|
return cls._load_values(_NEAT_ERRORS_BY_NAME[issue_type], args)
|
|
159
152
|
elif issue_type in _NEAT_WARNINGS_BY_NAME:
|
|
@@ -210,212 +203,42 @@ class NeatIssue:
|
|
|
210
203
|
return NotImplemented
|
|
211
204
|
return (type(self).__name__, self.as_message()) == (type(other).__name__, other.as_message())
|
|
212
205
|
|
|
206
|
+
def __str__(self) -> str:
|
|
207
|
+
return self.as_message()
|
|
208
|
+
|
|
213
209
|
|
|
214
210
|
@dataclass(unsafe_hash=True)
|
|
215
211
|
class NeatError(NeatIssue, Exception):
|
|
216
212
|
"""This is the base class for all exceptions (errors) used in Neat."""
|
|
217
213
|
|
|
218
|
-
|
|
219
|
-
def from_errors(cls, errors: "list[ErrorDetails | NeatError]", **kwargs) -> "list[NeatError]":
|
|
220
|
-
"""Convert a list of pydantic errors to a list of Error instances.
|
|
221
|
-
|
|
222
|
-
This is intended to be overridden in subclasses to handle specific error types.
|
|
223
|
-
"""
|
|
224
|
-
all_errors: list[NeatError] = []
|
|
225
|
-
read_info_by_sheet = kwargs.get("read_info_by_sheet")
|
|
226
|
-
|
|
227
|
-
for error in errors:
|
|
228
|
-
if (
|
|
229
|
-
isinstance(error, dict)
|
|
230
|
-
and error["type"] == "is_instance_of"
|
|
231
|
-
and error["loc"][1] == "is-instance[SheetList]"
|
|
232
|
-
):
|
|
233
|
-
# Skip the error for SheetList, as it is not relevant for the user. This is an
|
|
234
|
-
# internal class used to have helper methods for a lists as .to_pandas()
|
|
235
|
-
continue
|
|
236
|
-
|
|
237
|
-
neat_error: NeatError | None = None
|
|
238
|
-
if isinstance(error, dict) and isinstance(ctx := error.get("ctx"), dict) and "error" in ctx:
|
|
239
|
-
neat_error = ctx["error"]
|
|
240
|
-
elif isinstance(error, NeatError | MultiValueError):
|
|
241
|
-
neat_error = error
|
|
242
|
-
|
|
243
|
-
loc = error["loc"] if isinstance(error, dict) else tuple()
|
|
244
|
-
if isinstance(neat_error, MultiValueError):
|
|
245
|
-
all_errors.extend([cls._adjust_error(e, loc, read_info_by_sheet) for e in neat_error.errors])
|
|
246
|
-
elif isinstance(neat_error, NeatError):
|
|
247
|
-
all_errors.append(cls._adjust_error(neat_error, loc, read_info_by_sheet))
|
|
248
|
-
elif isinstance(error, dict) and len(loc) >= 4 and read_info_by_sheet:
|
|
249
|
-
all_errors.append(RowError.from_pydantic_error(error, read_info_by_sheet))
|
|
250
|
-
elif isinstance(error, dict):
|
|
251
|
-
all_errors.append(DefaultPydanticError.from_pydantic_error(error))
|
|
252
|
-
else:
|
|
253
|
-
# This is unreachable. However, in case it turns out to be reachable, we want to know about it.
|
|
254
|
-
raise ValueError(f"Unsupported error type: {error}")
|
|
255
|
-
return all_errors
|
|
256
|
-
|
|
257
|
-
@classmethod
|
|
258
|
-
def _adjust_error(
|
|
259
|
-
cls, error: "NeatError", loc: tuple[str | int, ...], read_info_by_sheet: dict[str, SpreadsheetRead] | None
|
|
260
|
-
) -> "NeatError":
|
|
261
|
-
from .errors._wrapper import MetadataValueError
|
|
262
|
-
|
|
263
|
-
if read_info_by_sheet:
|
|
264
|
-
cls._adjust_row_numbers(error, read_info_by_sheet)
|
|
265
|
-
if len(loc) == 2 and isinstance(loc[0], str) and loc[0].casefold() == "metadata":
|
|
266
|
-
return MetadataValueError(field_name=str(loc[1]), error=error)
|
|
267
|
-
return error
|
|
268
|
-
|
|
269
|
-
@staticmethod
|
|
270
|
-
def _adjust_row_numbers(caught_error: "NeatError", read_info_by_sheet: dict[str, SpreadsheetRead]) -> None:
|
|
271
|
-
from cognite.neat._issues.errors._properties import PropertyDefinitionDuplicatedError
|
|
272
|
-
from cognite.neat._issues.errors._resources import ResourceNotDefinedError
|
|
273
|
-
|
|
274
|
-
reader = read_info_by_sheet.get("Properties", SpreadsheetRead())
|
|
275
|
-
|
|
276
|
-
if isinstance(caught_error, PropertyDefinitionDuplicatedError) and caught_error.location_name == "rows":
|
|
277
|
-
adjusted_row_number = (
|
|
278
|
-
tuple(
|
|
279
|
-
reader.adjusted_row_number(row_no) if isinstance(row_no, int) else row_no
|
|
280
|
-
for row_no in caught_error.locations or []
|
|
281
|
-
)
|
|
282
|
-
or None
|
|
283
|
-
)
|
|
284
|
-
# The error is frozen, so we have to use __setattr__ to change the row number
|
|
285
|
-
object.__setattr__(caught_error, "locations", adjusted_row_number)
|
|
286
|
-
elif isinstance(caught_error, RowError):
|
|
287
|
-
# Adjusting the row number to the actual row number in the spreadsheet
|
|
288
|
-
new_row = reader.adjusted_row_number(caught_error.row)
|
|
289
|
-
# The error is frozen, so we have to use __setattr__ to change the row number
|
|
290
|
-
object.__setattr__(caught_error, "row", new_row)
|
|
291
|
-
elif isinstance(caught_error, ResourceNotDefinedError):
|
|
292
|
-
if isinstance(caught_error.row_number, int) and caught_error.sheet_name == "Properties":
|
|
293
|
-
new_row = reader.adjusted_row_number(caught_error.row_number)
|
|
294
|
-
object.__setattr__(caught_error, "row_number", new_row)
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
@dataclass(unsafe_hash=True)
|
|
298
|
-
class DefaultPydanticError(NeatError, ValueError):
|
|
299
|
-
"""{type}: {msg} [loc={loc}]"""
|
|
300
|
-
|
|
301
|
-
type: str
|
|
302
|
-
loc: tuple[int | str, ...]
|
|
303
|
-
msg: str
|
|
304
|
-
|
|
305
|
-
@classmethod
|
|
306
|
-
def from_pydantic_error(cls, error: ErrorDetails) -> "NeatError":
|
|
307
|
-
loc = error["loc"]
|
|
308
|
-
if len(loc) >= 2 and isinstance(loc[0], str) and loc[0].casefold() == "metadata":
|
|
309
|
-
from .errors._general import NeatValueError
|
|
310
|
-
from .errors._wrapper import MetadataValueError
|
|
311
|
-
|
|
312
|
-
return MetadataValueError(
|
|
313
|
-
field_name=str(loc[1]), error=NeatValueError(f"{error['msg']} got '{error['input']}'")
|
|
314
|
-
)
|
|
315
|
-
|
|
316
|
-
return cls(
|
|
317
|
-
type=error["type"],
|
|
318
|
-
loc=error["loc"],
|
|
319
|
-
msg=error["msg"],
|
|
320
|
-
)
|
|
321
|
-
|
|
322
|
-
def as_message(self, include_type: bool = True) -> str:
|
|
323
|
-
if self.loc and len(self.loc) == 1:
|
|
324
|
-
return f"{self.loc[0]} sheet: {self.msg}"
|
|
325
|
-
elif self.loc and len(self.loc) == 2:
|
|
326
|
-
return f"{self.loc[0]} sheet field/column <{self.loc[1]}>: {self.msg}"
|
|
327
|
-
else:
|
|
328
|
-
return self.msg
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
@dataclass(unsafe_hash=True)
|
|
332
|
-
class RowError(NeatError, ValueError):
|
|
333
|
-
"""In {sheet_name}, row={row}, column={column}: {msg}. [type={type}, input_value={input}]"""
|
|
334
|
-
|
|
335
|
-
extra = "For further information visit {url}"
|
|
336
|
-
|
|
337
|
-
sheet_name: str
|
|
338
|
-
column: str
|
|
339
|
-
row: int
|
|
340
|
-
type: str
|
|
341
|
-
msg: str
|
|
342
|
-
input: Any
|
|
343
|
-
url: str | None = None
|
|
344
|
-
|
|
345
|
-
@classmethod
|
|
346
|
-
def from_pydantic_error(
|
|
347
|
-
cls,
|
|
348
|
-
error: ErrorDetails,
|
|
349
|
-
read_info_by_sheet: dict[str, SpreadsheetRead] | None = None,
|
|
350
|
-
) -> Self:
|
|
351
|
-
sheet_name, _, row, column, *__ = error["loc"]
|
|
352
|
-
reader = (read_info_by_sheet or {}).get(str(sheet_name), SpreadsheetRead())
|
|
353
|
-
return cls(
|
|
354
|
-
sheet_name=str(sheet_name),
|
|
355
|
-
column=str(column),
|
|
356
|
-
row=reader.adjusted_row_number(int(row)),
|
|
357
|
-
type=error["type"],
|
|
358
|
-
msg=error["msg"],
|
|
359
|
-
input=error.get("input"),
|
|
360
|
-
url=str(url) if (url := error.get("url")) else None,
|
|
361
|
-
)
|
|
362
|
-
|
|
363
|
-
def as_message(self, include_type: bool = True) -> str:
|
|
364
|
-
input_str = str(self.input) if self.input is not None else ""
|
|
365
|
-
input_str = input_str[:50] + "..." if len(input_str) > 50 else input_str
|
|
366
|
-
output = (
|
|
367
|
-
f"In {self.sheet_name}, row={self.row}, column={self.column}: {self.msg}. "
|
|
368
|
-
f"[type={self.type}, input_value={input_str}]"
|
|
369
|
-
)
|
|
370
|
-
if self.url:
|
|
371
|
-
output += f" For further information visit {self.url}"
|
|
372
|
-
return output
|
|
214
|
+
...
|
|
373
215
|
|
|
374
216
|
|
|
375
217
|
@dataclass(unsafe_hash=True)
|
|
376
218
|
class NeatWarning(NeatIssue, UserWarning):
|
|
377
219
|
"""This is the base class for all warnings used in Neat."""
|
|
378
220
|
|
|
379
|
-
|
|
380
|
-
def from_warning(cls, warning: WarningMessage) -> "NeatWarning":
|
|
381
|
-
"""Create a NeatWarning from a WarningMessage."""
|
|
382
|
-
return DefaultWarning.from_warning_message(warning)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
@dataclass(unsafe_hash=True)
|
|
386
|
-
class DefaultWarning(NeatWarning):
|
|
387
|
-
"""{category}: {warning}"""
|
|
388
|
-
|
|
389
|
-
extra = "Source: {source}"
|
|
221
|
+
...
|
|
390
222
|
|
|
391
|
-
warning: str
|
|
392
|
-
category: str
|
|
393
|
-
source: str | None = None
|
|
394
223
|
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
if isinstance(warning.message, NeatWarning):
|
|
398
|
-
return warning.message
|
|
399
|
-
|
|
400
|
-
return cls(
|
|
401
|
-
warning=str(warning.message),
|
|
402
|
-
category=warning.category.__name__,
|
|
403
|
-
source=warning.source,
|
|
404
|
-
)
|
|
224
|
+
class MultiValueError(ValueError):
|
|
225
|
+
"""This is a container for multiple errors.
|
|
405
226
|
|
|
406
|
-
|
|
407
|
-
|
|
227
|
+
It is used in the pydantic field_validator/model_validator to collect multiple errors, which
|
|
228
|
+
can then be caught in a try-except block and returned as an IssueList.
|
|
408
229
|
|
|
230
|
+
"""
|
|
409
231
|
|
|
410
|
-
|
|
232
|
+
def __init__(self, errors: Sequence[NeatIssue]):
|
|
233
|
+
self.errors = IssueList(errors)
|
|
411
234
|
|
|
412
235
|
|
|
413
|
-
class
|
|
236
|
+
class IssueList(list, Sequence[NeatIssue]):
|
|
414
237
|
"""This is a generic list of NeatIssues."""
|
|
415
238
|
|
|
416
239
|
def __init__(
|
|
417
240
|
self,
|
|
418
|
-
issues: Sequence[
|
|
241
|
+
issues: Sequence[NeatIssue] | None = None,
|
|
419
242
|
title: str | None = None,
|
|
420
243
|
action: str | None = None,
|
|
421
244
|
hint: str | None = None,
|
|
@@ -462,36 +285,17 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
|
|
|
462
285
|
|
|
463
286
|
def trigger_warnings(self) -> None:
|
|
464
287
|
"""Trigger all warnings in this list."""
|
|
465
|
-
for warning in
|
|
288
|
+
for warning in self.warnings:
|
|
466
289
|
warnings.warn(warning, stacklevel=2)
|
|
467
290
|
|
|
468
291
|
def to_pandas(self) -> pd.DataFrame:
|
|
469
292
|
"""Return a pandas DataFrame representation of this list."""
|
|
470
293
|
return pd.DataFrame([issue.dump() for issue in self])
|
|
471
294
|
|
|
472
|
-
def
|
|
473
|
-
return self.to_pandas()._repr_html_() # type: ignore[operator]
|
|
474
|
-
|
|
475
|
-
def as_exception(self) -> "MultiValueError":
|
|
295
|
+
def as_exception(self) -> MultiValueError:
|
|
476
296
|
"""Return a MultiValueError with all the errors in this list."""
|
|
477
297
|
return MultiValueError(self.errors)
|
|
478
298
|
|
|
479
|
-
|
|
480
|
-
class MultiValueError(ValueError):
|
|
481
|
-
"""This is a container for multiple errors.
|
|
482
|
-
|
|
483
|
-
It is used in the pydantic field_validator/model_validator to collect multiple errors, which
|
|
484
|
-
can then be caught in a try-except block and returned as an IssueList.
|
|
485
|
-
|
|
486
|
-
"""
|
|
487
|
-
|
|
488
|
-
def __init__(self, errors: Sequence[NeatIssue]):
|
|
489
|
-
self.errors = list(errors)
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
class IssueList(NeatIssueList[NeatIssue]):
|
|
493
|
-
"""This is a list of NeatIssues."""
|
|
494
|
-
|
|
495
299
|
def _repr_html_(self) -> str | None:
|
|
496
300
|
if self.action and not self:
|
|
497
301
|
header = f"Success: {self.action}"
|
|
@@ -530,38 +334,3 @@ def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[t
|
|
|
530
334
|
for s in cls_.__subclasses__():
|
|
531
335
|
yield s
|
|
532
336
|
yield from _get_subclasses(s, False)
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
@contextmanager
|
|
536
|
-
def catch_warnings() -> Iterator[IssueList]:
|
|
537
|
-
"""Catch warnings and append them to the issues list."""
|
|
538
|
-
issues = IssueList()
|
|
539
|
-
with warnings.catch_warnings(record=True) as warning_logger:
|
|
540
|
-
warnings.simplefilter("always")
|
|
541
|
-
try:
|
|
542
|
-
yield issues
|
|
543
|
-
finally:
|
|
544
|
-
if warning_logger:
|
|
545
|
-
issues.extend([NeatWarning.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
@contextmanager
|
|
549
|
-
def catch_issues(error_args: dict[str, Any] | None = None) -> Iterator[IssueList]:
|
|
550
|
-
"""This is an internal help function to handle issues and warnings.
|
|
551
|
-
|
|
552
|
-
Args:
|
|
553
|
-
error_args: Additional arguments to pass to the error class. The only use case as of (2025-01-03) is to pass
|
|
554
|
-
the read_info_by_sheet to the error class such that the row numbers can be adjusted to match the source
|
|
555
|
-
spreadsheet.
|
|
556
|
-
|
|
557
|
-
Returns:
|
|
558
|
-
IssueList: The list of issues.
|
|
559
|
-
|
|
560
|
-
"""
|
|
561
|
-
with catch_warnings() as issues:
|
|
562
|
-
try:
|
|
563
|
-
yield issues
|
|
564
|
-
except ValidationError as e:
|
|
565
|
-
issues.extend(NeatError.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
|
|
566
|
-
except (NeatError, MultiValueError) as e:
|
|
567
|
-
issues.extend(NeatError.from_errors([e], **(error_args or {}))) # type: ignore[arg-type, list-item]
|