cognite-neat 0.109.4__py3-none-any.whl → 0.111.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +8 -0
- cognite/neat/_client/_api/schema.py +43 -1
- cognite/neat/_client/data_classes/schema.py +4 -4
- cognite/neat/_constants.py +15 -1
- cognite/neat/_graph/extractors/__init__.py +4 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +8 -16
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +48 -19
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +23 -17
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +15 -17
- cognite/neat/_graph/extractors/_dict.py +102 -0
- cognite/neat/_graph/extractors/_dms.py +27 -40
- cognite/neat/_graph/extractors/_dms_graph.py +30 -3
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/extractors/_raw.py +67 -0
- cognite/neat/_graph/loaders/_base.py +20 -4
- cognite/neat/_graph/loaders/_rdf2dms.py +476 -383
- cognite/neat/_graph/queries/_base.py +163 -133
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +6 -22
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +69 -0
- cognite/neat/_issues/errors/__init__.py +20 -4
- cognite/neat/_issues/errors/_external.py +7 -0
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_issues/warnings/user_modeling.py +12 -0
- cognite/neat/_rules/_constants.py +12 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +430 -259
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +3 -9
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +66 -26
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/_base_rules.py +0 -2
- cognite/neat/_rules/models/data_types.py +7 -0
- cognite/neat/_rules/models/dms/_exporter.py +9 -8
- cognite/neat/_rules/models/dms/_rules.py +29 -2
- cognite/neat/_rules/models/dms/_rules_input.py +9 -1
- cognite/neat/_rules/models/dms/_validation.py +115 -5
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/entities/_single_value.py +8 -3
- cognite/neat/_rules/models/entities/_wrapped.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +3 -1
- cognite/neat/_rules/models/information/_validation.py +66 -17
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +234 -44
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +6 -4
- cognite/neat/_session/_explore.py +39 -0
- cognite/neat/_session/_inspect.py +25 -6
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +88 -20
- cognite/neat/_session/_set.py +7 -1
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +6 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +177 -19
- cognite/neat/_store/_graph_store.py +9 -246
- cognite/neat/_utils/rdf_.py +36 -5
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +124 -37
- cognite/neat/_utils/upload.py +2 -0
- cognite/neat/_version.py +2 -2
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/RECORD +83 -82
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/entry_points.txt +0 -0
cognite/neat/_issues/_base.py
CHANGED
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import sys
|
|
3
3
|
import warnings
|
|
4
|
-
from abc import
|
|
5
|
-
from collections.abc import Collection, Hashable, Iterable, Iterator, Sequence
|
|
6
|
-
from contextlib import contextmanager
|
|
4
|
+
from collections.abc import Collection, Hashable, Iterable, Sequence
|
|
7
5
|
from dataclasses import dataclass, fields
|
|
8
6
|
from functools import total_ordering
|
|
9
7
|
from pathlib import Path
|
|
10
8
|
from types import UnionType
|
|
11
9
|
from typing import Any, ClassVar, Literal, TypeAlias, TypeVar, get_args, get_origin
|
|
12
|
-
from warnings import WarningMessage
|
|
13
10
|
|
|
14
11
|
import pandas as pd
|
|
15
12
|
from cognite.client.data_classes.data_modeling import (
|
|
@@ -18,11 +15,8 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
18
15
|
PropertyId,
|
|
19
16
|
ViewId,
|
|
20
17
|
)
|
|
21
|
-
from pydantic import ValidationError
|
|
22
|
-
from pydantic_core import ErrorDetails
|
|
23
18
|
|
|
24
|
-
from cognite.neat._utils.
|
|
25
|
-
from cognite.neat._utils.text import humanize_collection, to_camel, to_snake
|
|
19
|
+
from cognite.neat._utils.text import humanize_collection, to_camel_case, to_snake_case
|
|
26
20
|
|
|
27
21
|
if sys.version_info < (3, 11):
|
|
28
22
|
from exceptiongroup import ExceptionGroup
|
|
@@ -32,11 +26,10 @@ else:
|
|
|
32
26
|
|
|
33
27
|
|
|
34
28
|
__all__ = [
|
|
35
|
-
"
|
|
29
|
+
"IssueList",
|
|
36
30
|
"MultiValueError",
|
|
37
31
|
"NeatError",
|
|
38
32
|
"NeatIssue",
|
|
39
|
-
"NeatIssueList",
|
|
40
33
|
"NeatWarning",
|
|
41
34
|
]
|
|
42
35
|
|
|
@@ -113,7 +106,7 @@ class NeatIssue:
|
|
|
113
106
|
"""Return a dictionary representation of the issue."""
|
|
114
107
|
variables = vars(self)
|
|
115
108
|
output = {
|
|
116
|
-
|
|
109
|
+
to_camel_case(key): self._dump_value(value)
|
|
117
110
|
for key, value in variables.items()
|
|
118
111
|
if not (value is None or key.startswith("_"))
|
|
119
112
|
}
|
|
@@ -153,7 +146,7 @@ class NeatIssue:
|
|
|
153
146
|
if "NeatIssue" not in data:
|
|
154
147
|
raise NeatValueError("The data does not contain a NeatIssue key.")
|
|
155
148
|
issue_type = data.pop("NeatIssue")
|
|
156
|
-
args = {
|
|
149
|
+
args = {to_snake_case(key): value for key, value in data.items()}
|
|
157
150
|
if issue_type in _NEAT_ERRORS_BY_NAME:
|
|
158
151
|
return cls._load_values(_NEAT_ERRORS_BY_NAME[issue_type], args)
|
|
159
152
|
elif issue_type in _NEAT_WARNINGS_BY_NAME:
|
|
@@ -210,212 +203,42 @@ class NeatIssue:
|
|
|
210
203
|
return NotImplemented
|
|
211
204
|
return (type(self).__name__, self.as_message()) == (type(other).__name__, other.as_message())
|
|
212
205
|
|
|
206
|
+
def __str__(self) -> str:
|
|
207
|
+
return self.as_message()
|
|
208
|
+
|
|
213
209
|
|
|
214
210
|
@dataclass(unsafe_hash=True)
|
|
215
211
|
class NeatError(NeatIssue, Exception):
|
|
216
212
|
"""This is the base class for all exceptions (errors) used in Neat."""
|
|
217
213
|
|
|
218
|
-
|
|
219
|
-
def from_errors(cls, errors: "list[ErrorDetails | NeatError]", **kwargs) -> "list[NeatError]":
|
|
220
|
-
"""Convert a list of pydantic errors to a list of Error instances.
|
|
221
|
-
|
|
222
|
-
This is intended to be overridden in subclasses to handle specific error types.
|
|
223
|
-
"""
|
|
224
|
-
all_errors: list[NeatError] = []
|
|
225
|
-
read_info_by_sheet = kwargs.get("read_info_by_sheet")
|
|
226
|
-
|
|
227
|
-
for error in errors:
|
|
228
|
-
if (
|
|
229
|
-
isinstance(error, dict)
|
|
230
|
-
and error["type"] == "is_instance_of"
|
|
231
|
-
and error["loc"][1] == "is-instance[SheetList]"
|
|
232
|
-
):
|
|
233
|
-
# Skip the error for SheetList, as it is not relevant for the user. This is an
|
|
234
|
-
# internal class used to have helper methods for a lists as .to_pandas()
|
|
235
|
-
continue
|
|
236
|
-
|
|
237
|
-
neat_error: NeatError | None = None
|
|
238
|
-
if isinstance(error, dict) and isinstance(ctx := error.get("ctx"), dict) and "error" in ctx:
|
|
239
|
-
neat_error = ctx["error"]
|
|
240
|
-
elif isinstance(error, NeatError | MultiValueError):
|
|
241
|
-
neat_error = error
|
|
242
|
-
|
|
243
|
-
loc = error["loc"] if isinstance(error, dict) else tuple()
|
|
244
|
-
if isinstance(neat_error, MultiValueError):
|
|
245
|
-
all_errors.extend([cls._adjust_error(e, loc, read_info_by_sheet) for e in neat_error.errors])
|
|
246
|
-
elif isinstance(neat_error, NeatError):
|
|
247
|
-
all_errors.append(cls._adjust_error(neat_error, loc, read_info_by_sheet))
|
|
248
|
-
elif isinstance(error, dict) and len(loc) >= 4 and read_info_by_sheet:
|
|
249
|
-
all_errors.append(RowError.from_pydantic_error(error, read_info_by_sheet))
|
|
250
|
-
elif isinstance(error, dict):
|
|
251
|
-
all_errors.append(DefaultPydanticError.from_pydantic_error(error))
|
|
252
|
-
else:
|
|
253
|
-
# This is unreachable. However, in case it turns out to be reachable, we want to know about it.
|
|
254
|
-
raise ValueError(f"Unsupported error type: {error}")
|
|
255
|
-
return all_errors
|
|
256
|
-
|
|
257
|
-
@classmethod
|
|
258
|
-
def _adjust_error(
|
|
259
|
-
cls, error: "NeatError", loc: tuple[str | int, ...], read_info_by_sheet: dict[str, SpreadsheetRead] | None
|
|
260
|
-
) -> "NeatError":
|
|
261
|
-
from .errors._wrapper import MetadataValueError
|
|
262
|
-
|
|
263
|
-
if read_info_by_sheet:
|
|
264
|
-
cls._adjust_row_numbers(error, read_info_by_sheet)
|
|
265
|
-
if len(loc) == 2 and isinstance(loc[0], str) and loc[0].casefold() == "metadata":
|
|
266
|
-
return MetadataValueError(field_name=str(loc[1]), error=error)
|
|
267
|
-
return error
|
|
268
|
-
|
|
269
|
-
@staticmethod
|
|
270
|
-
def _adjust_row_numbers(caught_error: "NeatError", read_info_by_sheet: dict[str, SpreadsheetRead]) -> None:
|
|
271
|
-
from cognite.neat._issues.errors._properties import PropertyDefinitionDuplicatedError
|
|
272
|
-
from cognite.neat._issues.errors._resources import ResourceNotDefinedError
|
|
273
|
-
|
|
274
|
-
reader = read_info_by_sheet.get("Properties", SpreadsheetRead())
|
|
275
|
-
|
|
276
|
-
if isinstance(caught_error, PropertyDefinitionDuplicatedError) and caught_error.location_name == "rows":
|
|
277
|
-
adjusted_row_number = (
|
|
278
|
-
tuple(
|
|
279
|
-
reader.adjusted_row_number(row_no) if isinstance(row_no, int) else row_no
|
|
280
|
-
for row_no in caught_error.locations or []
|
|
281
|
-
)
|
|
282
|
-
or None
|
|
283
|
-
)
|
|
284
|
-
# The error is frozen, so we have to use __setattr__ to change the row number
|
|
285
|
-
object.__setattr__(caught_error, "locations", adjusted_row_number)
|
|
286
|
-
elif isinstance(caught_error, RowError):
|
|
287
|
-
# Adjusting the row number to the actual row number in the spreadsheet
|
|
288
|
-
new_row = reader.adjusted_row_number(caught_error.row)
|
|
289
|
-
# The error is frozen, so we have to use __setattr__ to change the row number
|
|
290
|
-
object.__setattr__(caught_error, "row", new_row)
|
|
291
|
-
elif isinstance(caught_error, ResourceNotDefinedError):
|
|
292
|
-
if isinstance(caught_error.row_number, int) and caught_error.sheet_name == "Properties":
|
|
293
|
-
new_row = reader.adjusted_row_number(caught_error.row_number)
|
|
294
|
-
object.__setattr__(caught_error, "row_number", new_row)
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
@dataclass(unsafe_hash=True)
|
|
298
|
-
class DefaultPydanticError(NeatError, ValueError):
|
|
299
|
-
"""{type}: {msg} [loc={loc}]"""
|
|
300
|
-
|
|
301
|
-
type: str
|
|
302
|
-
loc: tuple[int | str, ...]
|
|
303
|
-
msg: str
|
|
304
|
-
|
|
305
|
-
@classmethod
|
|
306
|
-
def from_pydantic_error(cls, error: ErrorDetails) -> "NeatError":
|
|
307
|
-
loc = error["loc"]
|
|
308
|
-
if len(loc) >= 2 and isinstance(loc[0], str) and loc[0].casefold() == "metadata":
|
|
309
|
-
from .errors._general import NeatValueError
|
|
310
|
-
from .errors._wrapper import MetadataValueError
|
|
311
|
-
|
|
312
|
-
return MetadataValueError(
|
|
313
|
-
field_name=str(loc[1]), error=NeatValueError(f"{error['msg']} got '{error['input']}'")
|
|
314
|
-
)
|
|
315
|
-
|
|
316
|
-
return cls(
|
|
317
|
-
type=error["type"],
|
|
318
|
-
loc=error["loc"],
|
|
319
|
-
msg=error["msg"],
|
|
320
|
-
)
|
|
321
|
-
|
|
322
|
-
def as_message(self, include_type: bool = True) -> str:
|
|
323
|
-
if self.loc and len(self.loc) == 1:
|
|
324
|
-
return f"{self.loc[0]} sheet: {self.msg}"
|
|
325
|
-
elif self.loc and len(self.loc) == 2:
|
|
326
|
-
return f"{self.loc[0]} sheet field/column <{self.loc[1]}>: {self.msg}"
|
|
327
|
-
else:
|
|
328
|
-
return self.msg
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
@dataclass(unsafe_hash=True)
|
|
332
|
-
class RowError(NeatError, ValueError):
|
|
333
|
-
"""In {sheet_name}, row={row}, column={column}: {msg}. [type={type}, input_value={input}]"""
|
|
334
|
-
|
|
335
|
-
extra = "For further information visit {url}"
|
|
336
|
-
|
|
337
|
-
sheet_name: str
|
|
338
|
-
column: str
|
|
339
|
-
row: int
|
|
340
|
-
type: str
|
|
341
|
-
msg: str
|
|
342
|
-
input: Any
|
|
343
|
-
url: str | None = None
|
|
344
|
-
|
|
345
|
-
@classmethod
|
|
346
|
-
def from_pydantic_error(
|
|
347
|
-
cls,
|
|
348
|
-
error: ErrorDetails,
|
|
349
|
-
read_info_by_sheet: dict[str, SpreadsheetRead] | None = None,
|
|
350
|
-
) -> Self:
|
|
351
|
-
sheet_name, _, row, column, *__ = error["loc"]
|
|
352
|
-
reader = (read_info_by_sheet or {}).get(str(sheet_name), SpreadsheetRead())
|
|
353
|
-
return cls(
|
|
354
|
-
sheet_name=str(sheet_name),
|
|
355
|
-
column=str(column),
|
|
356
|
-
row=reader.adjusted_row_number(int(row)),
|
|
357
|
-
type=error["type"],
|
|
358
|
-
msg=error["msg"],
|
|
359
|
-
input=error.get("input"),
|
|
360
|
-
url=str(url) if (url := error.get("url")) else None,
|
|
361
|
-
)
|
|
362
|
-
|
|
363
|
-
def as_message(self, include_type: bool = True) -> str:
|
|
364
|
-
input_str = str(self.input) if self.input is not None else ""
|
|
365
|
-
input_str = input_str[:50] + "..." if len(input_str) > 50 else input_str
|
|
366
|
-
output = (
|
|
367
|
-
f"In {self.sheet_name}, row={self.row}, column={self.column}: {self.msg}. "
|
|
368
|
-
f"[type={self.type}, input_value={input_str}]"
|
|
369
|
-
)
|
|
370
|
-
if self.url:
|
|
371
|
-
output += f" For further information visit {self.url}"
|
|
372
|
-
return output
|
|
214
|
+
...
|
|
373
215
|
|
|
374
216
|
|
|
375
217
|
@dataclass(unsafe_hash=True)
|
|
376
218
|
class NeatWarning(NeatIssue, UserWarning):
|
|
377
219
|
"""This is the base class for all warnings used in Neat."""
|
|
378
220
|
|
|
379
|
-
|
|
380
|
-
def from_warning(cls, warning: WarningMessage) -> "NeatWarning":
|
|
381
|
-
"""Create a NeatWarning from a WarningMessage."""
|
|
382
|
-
return DefaultWarning.from_warning_message(warning)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
@dataclass(unsafe_hash=True)
|
|
386
|
-
class DefaultWarning(NeatWarning):
|
|
387
|
-
"""{category}: {warning}"""
|
|
388
|
-
|
|
389
|
-
extra = "Source: {source}"
|
|
221
|
+
...
|
|
390
222
|
|
|
391
|
-
warning: str
|
|
392
|
-
category: str
|
|
393
|
-
source: str | None = None
|
|
394
223
|
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
if isinstance(warning.message, NeatWarning):
|
|
398
|
-
return warning.message
|
|
399
|
-
|
|
400
|
-
return cls(
|
|
401
|
-
warning=str(warning.message),
|
|
402
|
-
category=warning.category.__name__,
|
|
403
|
-
source=warning.source,
|
|
404
|
-
)
|
|
224
|
+
class MultiValueError(ValueError):
|
|
225
|
+
"""This is a container for multiple errors.
|
|
405
226
|
|
|
406
|
-
|
|
407
|
-
|
|
227
|
+
It is used in the pydantic field_validator/model_validator to collect multiple errors, which
|
|
228
|
+
can then be caught in a try-except block and returned as an IssueList.
|
|
408
229
|
|
|
230
|
+
"""
|
|
409
231
|
|
|
410
|
-
|
|
232
|
+
def __init__(self, errors: Sequence[NeatIssue]):
|
|
233
|
+
self.errors = IssueList(errors)
|
|
411
234
|
|
|
412
235
|
|
|
413
|
-
class
|
|
236
|
+
class IssueList(list, Sequence[NeatIssue]):
|
|
414
237
|
"""This is a generic list of NeatIssues."""
|
|
415
238
|
|
|
416
239
|
def __init__(
|
|
417
240
|
self,
|
|
418
|
-
issues: Sequence[
|
|
241
|
+
issues: Sequence[NeatIssue] | None = None,
|
|
419
242
|
title: str | None = None,
|
|
420
243
|
action: str | None = None,
|
|
421
244
|
hint: str | None = None,
|
|
@@ -462,36 +285,17 @@ class NeatIssueList(list, Sequence[T_NeatIssue], ABC):
|
|
|
462
285
|
|
|
463
286
|
def trigger_warnings(self) -> None:
|
|
464
287
|
"""Trigger all warnings in this list."""
|
|
465
|
-
for warning in
|
|
288
|
+
for warning in self.warnings:
|
|
466
289
|
warnings.warn(warning, stacklevel=2)
|
|
467
290
|
|
|
468
291
|
def to_pandas(self) -> pd.DataFrame:
|
|
469
292
|
"""Return a pandas DataFrame representation of this list."""
|
|
470
293
|
return pd.DataFrame([issue.dump() for issue in self])
|
|
471
294
|
|
|
472
|
-
def
|
|
473
|
-
return self.to_pandas()._repr_html_() # type: ignore[operator]
|
|
474
|
-
|
|
475
|
-
def as_exception(self) -> "MultiValueError":
|
|
295
|
+
def as_exception(self) -> MultiValueError:
|
|
476
296
|
"""Return a MultiValueError with all the errors in this list."""
|
|
477
297
|
return MultiValueError(self.errors)
|
|
478
298
|
|
|
479
|
-
|
|
480
|
-
class MultiValueError(ValueError):
|
|
481
|
-
"""This is a container for multiple errors.
|
|
482
|
-
|
|
483
|
-
It is used in the pydantic field_validator/model_validator to collect multiple errors, which
|
|
484
|
-
can then be caught in a try-except block and returned as an IssueList.
|
|
485
|
-
|
|
486
|
-
"""
|
|
487
|
-
|
|
488
|
-
def __init__(self, errors: Sequence[NeatIssue]):
|
|
489
|
-
self.errors = list(errors)
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
class IssueList(NeatIssueList[NeatIssue]):
|
|
493
|
-
"""This is a list of NeatIssues."""
|
|
494
|
-
|
|
495
299
|
def _repr_html_(self) -> str | None:
|
|
496
300
|
if self.action and not self:
|
|
497
301
|
header = f"Success: {self.action}"
|
|
@@ -530,38 +334,3 @@ def _get_subclasses(cls_: type[T_Cls], include_base: bool = False) -> Iterable[t
|
|
|
530
334
|
for s in cls_.__subclasses__():
|
|
531
335
|
yield s
|
|
532
336
|
yield from _get_subclasses(s, False)
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
@contextmanager
|
|
536
|
-
def catch_warnings() -> Iterator[IssueList]:
|
|
537
|
-
"""Catch warnings and append them to the issues list."""
|
|
538
|
-
issues = IssueList()
|
|
539
|
-
with warnings.catch_warnings(record=True) as warning_logger:
|
|
540
|
-
warnings.simplefilter("always")
|
|
541
|
-
try:
|
|
542
|
-
yield issues
|
|
543
|
-
finally:
|
|
544
|
-
if warning_logger:
|
|
545
|
-
issues.extend([NeatWarning.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
@contextmanager
|
|
549
|
-
def catch_issues(error_args: dict[str, Any] | None = None) -> Iterator[IssueList]:
|
|
550
|
-
"""This is an internal help function to handle issues and warnings.
|
|
551
|
-
|
|
552
|
-
Args:
|
|
553
|
-
error_args: Additional arguments to pass to the error class. The only use case as of (2025-01-03) is to pass
|
|
554
|
-
the read_info_by_sheet to the error class such that the row numbers can be adjusted to match the source
|
|
555
|
-
spreadsheet.
|
|
556
|
-
|
|
557
|
-
Returns:
|
|
558
|
-
IssueList: The list of issues.
|
|
559
|
-
|
|
560
|
-
"""
|
|
561
|
-
with catch_warnings() as issues:
|
|
562
|
-
try:
|
|
563
|
-
yield issues
|
|
564
|
-
except ValidationError as e:
|
|
565
|
-
issues.extend(NeatError.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
|
|
566
|
-
except (NeatError, MultiValueError) as e:
|
|
567
|
-
issues.extend(NeatError.from_errors([e], **(error_args or {}))) # type: ignore[arg-type, list-item]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from collections.abc import Iterator
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
|
|
5
|
+
from pydantic import ValidationError
|
|
6
|
+
|
|
7
|
+
from cognite.neat._utils.spreadsheet import SpreadsheetRead
|
|
8
|
+
|
|
9
|
+
from ._base import IssueList, MultiValueError, NeatError
|
|
10
|
+
from ._factory import from_pydantic_errors, from_warning
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@contextmanager
|
|
14
|
+
def catch_warnings() -> Iterator[IssueList]:
|
|
15
|
+
"""Catch warnings and append them to the issues list."""
|
|
16
|
+
issues = IssueList()
|
|
17
|
+
with warnings.catch_warnings(record=True) as warning_logger:
|
|
18
|
+
warnings.simplefilter("always")
|
|
19
|
+
try:
|
|
20
|
+
yield issues
|
|
21
|
+
finally:
|
|
22
|
+
if warning_logger:
|
|
23
|
+
issues.extend([from_warning(warning) for warning in warning_logger])
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@contextmanager
|
|
27
|
+
def catch_issues(read_info_by_sheet: dict[str, SpreadsheetRead] | None = None) -> Iterator[IssueList]:
|
|
28
|
+
"""This is an internal help function to handle issues and warnings.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
read_info_by_sheet (dict[str, SpreadsheetRead]): The read information by sheet. This is used to adjust
|
|
32
|
+
the row numbers in the errors/warnings.
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
IssueList: The list of issues.
|
|
36
|
+
|
|
37
|
+
"""
|
|
38
|
+
with catch_warnings() as issues:
|
|
39
|
+
try:
|
|
40
|
+
yield issues
|
|
41
|
+
except ValidationError as e:
|
|
42
|
+
issues.extend(from_pydantic_errors(e.errors(), read_info_by_sheet))
|
|
43
|
+
except NeatError as single:
|
|
44
|
+
issues.append(single)
|
|
45
|
+
except MultiValueError as multi:
|
|
46
|
+
issues.extend(multi.errors)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
from warnings import WarningMessage
|
|
3
|
+
|
|
4
|
+
from pydantic_core import ErrorDetails
|
|
5
|
+
|
|
6
|
+
from cognite.neat._issues._base import NeatError, NeatWarning
|
|
7
|
+
from cognite.neat._utils.spreadsheet import SpreadsheetRead
|
|
8
|
+
|
|
9
|
+
from .errors import NeatValueError, SpreadsheetError
|
|
10
|
+
from .warnings import NeatValueWarning
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def from_pydantic_errors(
|
|
14
|
+
errors: list[ErrorDetails], read_info_by_sheet: dict[str, SpreadsheetRead] | None = None
|
|
15
|
+
) -> list[NeatError]:
|
|
16
|
+
read_info_by_sheet = read_info_by_sheet or {}
|
|
17
|
+
return [
|
|
18
|
+
_from_pydantic_error(error, read_info_by_sheet)
|
|
19
|
+
for error in errors
|
|
20
|
+
# Skip the error for SheetList, as it is not relevant for the user. This is an
|
|
21
|
+
# internal class used to have helper methods for a lists as .to_pandas()
|
|
22
|
+
if not (error["type"] == "is_instance_of" and error["loc"][1] == "is-instance[SheetList]")
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def from_warning(warning: WarningMessage) -> NeatWarning:
|
|
27
|
+
if isinstance(warning.message, NeatWarning):
|
|
28
|
+
return warning.message
|
|
29
|
+
message = f"{warning.category.__name__}: {warning.message!s}"
|
|
30
|
+
if warning.source:
|
|
31
|
+
message += f" Source: {warning.source}"
|
|
32
|
+
return NeatValueWarning(message)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _from_pydantic_error(error: ErrorDetails, read_info_by_sheet: dict[str, SpreadsheetRead]) -> NeatError:
|
|
36
|
+
neat_error = _create_neat_value_error(error)
|
|
37
|
+
location = error["loc"]
|
|
38
|
+
|
|
39
|
+
# only errors caused in model_validate will have location information
|
|
40
|
+
if location:
|
|
41
|
+
return SpreadsheetError.create(location, neat_error, read_info_by_sheet.get(cast(str, location[0])))
|
|
42
|
+
|
|
43
|
+
# errors that occur while for example parsing spreadsheet in input rules
|
|
44
|
+
# will not have location information so we return neat_error as is
|
|
45
|
+
# this is workaround until more elegant solution is found
|
|
46
|
+
return neat_error
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _create_neat_value_error(error: ErrorDetails) -> NeatValueError:
|
|
50
|
+
if (ctx := error.get("ctx")) and (neat_error := ctx.get("error")) and isinstance(neat_error, NeatError):
|
|
51
|
+
# Is already a NeatError
|
|
52
|
+
return neat_error
|
|
53
|
+
return _pydantic_to_neat_error(error)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _pydantic_to_neat_error(error: ErrorDetails) -> NeatValueError:
|
|
57
|
+
error_type = error["type"]
|
|
58
|
+
input_value = error["input"]
|
|
59
|
+
match error_type:
|
|
60
|
+
# See https://docs.pydantic.dev/latest/errors/validation_errors/ for all possible error types:
|
|
61
|
+
case error_type if error_type.endswith("_type") | error_type.endswith("_parsing"):
|
|
62
|
+
if input_value is None:
|
|
63
|
+
return NeatValueError("value is missing.")
|
|
64
|
+
expected_type = error_type.removesuffix("_type").removesuffix("_parsing")
|
|
65
|
+
return NeatValueError(f"Expected a {expected_type} type, got {input_value!r}")
|
|
66
|
+
case _:
|
|
67
|
+
# The above cases overwrite the human-readable message from pydantic.
|
|
68
|
+
# Motivation for overwriting is that pydantic is developer-oriented and while neat is SME-oriented.
|
|
69
|
+
return NeatValueError(f"{error['msg']} got '{input_value}'")
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
from cognite.neat._issues._base import
|
|
1
|
+
from cognite.neat._issues._base import NeatError, _get_subclasses
|
|
2
2
|
|
|
3
3
|
from ._external import (
|
|
4
4
|
AuthorizationError,
|
|
5
5
|
CDFMissingClientError,
|
|
6
|
+
CDFMissingResourcesError,
|
|
6
7
|
FileMissingRequiredFieldError,
|
|
7
8
|
FileNotAFileError,
|
|
8
9
|
FileNotFoundNeatError,
|
|
@@ -31,12 +32,24 @@ from ._resources import (
|
|
|
31
32
|
ResourceNotFoundError,
|
|
32
33
|
ResourceRetrievalError,
|
|
33
34
|
)
|
|
34
|
-
from ._wrapper import
|
|
35
|
+
from ._wrapper import (
|
|
36
|
+
ClassValueError,
|
|
37
|
+
ContainerValueError,
|
|
38
|
+
EnumValueError,
|
|
39
|
+
MetadataValueError,
|
|
40
|
+
NodeValueError,
|
|
41
|
+
PropertyValueError,
|
|
42
|
+
SpreadsheetError,
|
|
43
|
+
ViewValueError,
|
|
44
|
+
)
|
|
35
45
|
|
|
36
46
|
__all__ = [
|
|
37
47
|
"AuthorizationError",
|
|
38
48
|
"CDFMissingClientError",
|
|
39
|
-
"
|
|
49
|
+
"CDFMissingResourcesError",
|
|
50
|
+
"ClassValueError",
|
|
51
|
+
"ContainerValueError",
|
|
52
|
+
"EnumValueError",
|
|
40
53
|
"FileMissingRequiredFieldError",
|
|
41
54
|
"FileNotAFileError",
|
|
42
55
|
"FileNotFoundNeatError",
|
|
@@ -48,12 +61,14 @@ __all__ = [
|
|
|
48
61
|
"NeatTypeError",
|
|
49
62
|
"NeatValueError",
|
|
50
63
|
"NeatYamlError",
|
|
64
|
+
"NodeValueError",
|
|
51
65
|
"OxigraphStorageLockedError",
|
|
52
66
|
"PropertyDefinitionDuplicatedError",
|
|
53
67
|
"PropertyDefinitionError",
|
|
54
68
|
"PropertyMappingDuplicatedError",
|
|
55
69
|
"PropertyNotFoundError",
|
|
56
70
|
"PropertyTypeNotSupportedError",
|
|
71
|
+
"PropertyValueError",
|
|
57
72
|
"RegexViolationError",
|
|
58
73
|
"ResourceChangedError",
|
|
59
74
|
"ResourceConversionError",
|
|
@@ -65,7 +80,8 @@ __all__ = [
|
|
|
65
80
|
"ResourceNotFoundError",
|
|
66
81
|
"ResourceRetrievalError",
|
|
67
82
|
"ReversedConnectionNotFeasibleError",
|
|
68
|
-
"
|
|
83
|
+
"SpreadsheetError",
|
|
84
|
+
"ViewValueError",
|
|
69
85
|
]
|
|
70
86
|
|
|
71
87
|
_NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}
|
|
@@ -80,3 +80,10 @@ class CDFMissingClientError(NeatError, RuntimeError):
|
|
|
80
80
|
"""CDF client is required: {reason}"""
|
|
81
81
|
|
|
82
82
|
reason: str
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(unsafe_hash=True)
|
|
86
|
+
class CDFMissingResourcesError(NeatError, RuntimeError):
|
|
87
|
+
"""Following CDF resources are missing: {resources}"""
|
|
88
|
+
|
|
89
|
+
resources: str
|
|
@@ -1,11 +1,89 @@
|
|
|
1
|
+
from abc import ABC
|
|
1
2
|
from dataclasses import dataclass
|
|
3
|
+
from typing import ClassVar, cast
|
|
2
4
|
|
|
3
5
|
from cognite.neat._issues import NeatError
|
|
6
|
+
from cognite.neat._utils.spreadsheet import SpreadsheetRead
|
|
4
7
|
|
|
5
8
|
|
|
6
9
|
@dataclass(unsafe_hash=True)
|
|
7
|
-
class
|
|
8
|
-
"""
|
|
10
|
+
class SpreadsheetError(NeatError, ValueError, ABC):
|
|
11
|
+
"""In row {row}: {error}"""
|
|
9
12
|
|
|
10
|
-
|
|
13
|
+
_name: ClassVar[str] = ""
|
|
11
14
|
error: NeatError
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def create(
|
|
18
|
+
cls, location: tuple[int | str, ...], error: NeatError, spreadsheet: SpreadsheetRead | None = None
|
|
19
|
+
) -> "SpreadsheetError":
|
|
20
|
+
spreadsheet_name = cast(str, location[0])
|
|
21
|
+
if spreadsheet_name not in ERROR_CLS_BY_SPREADSHEET_NAME:
|
|
22
|
+
# This happens for the metadata sheet, which are individual fields
|
|
23
|
+
return MetadataValueError(error, field_name=spreadsheet_name)
|
|
24
|
+
|
|
25
|
+
error_cls = ERROR_CLS_BY_SPREADSHEET_NAME[spreadsheet_name]
|
|
26
|
+
row, column = cast(tuple[int, str], location[2:4])
|
|
27
|
+
|
|
28
|
+
if spreadsheet:
|
|
29
|
+
row = spreadsheet.adjusted_row_number(row)
|
|
30
|
+
|
|
31
|
+
return error_cls(
|
|
32
|
+
row=row,
|
|
33
|
+
error=error,
|
|
34
|
+
column=column,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(unsafe_hash=True)
|
|
39
|
+
class SpreadsheetListError(SpreadsheetError, ABC):
|
|
40
|
+
"""In row {row}, column '{column}': {error}"""
|
|
41
|
+
|
|
42
|
+
row: int
|
|
43
|
+
column: str
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(unsafe_hash=True)
|
|
47
|
+
class MetadataValueError(SpreadsheetError):
|
|
48
|
+
"""In field {field_name}: {error}"""
|
|
49
|
+
|
|
50
|
+
_type: ClassVar[str] = "Metadata"
|
|
51
|
+
field_name: str
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(unsafe_hash=True)
|
|
55
|
+
class ViewValueError(SpreadsheetListError):
|
|
56
|
+
_name = "Views"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(unsafe_hash=True)
|
|
60
|
+
class ContainerValueError(SpreadsheetListError):
|
|
61
|
+
_name = "Containers"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(unsafe_hash=True)
|
|
65
|
+
class PropertyValueError(SpreadsheetListError):
|
|
66
|
+
_name = "Properties"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(unsafe_hash=True)
|
|
70
|
+
class ClassValueError(SpreadsheetListError):
|
|
71
|
+
_name = "Classes"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(unsafe_hash=True)
|
|
75
|
+
class EnumValueError(SpreadsheetListError):
|
|
76
|
+
_name = "Enum"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(unsafe_hash=True)
|
|
80
|
+
class NodeValueError(SpreadsheetListError):
|
|
81
|
+
_name = "Nodes"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
ERROR_CLS_BY_SPREADSHEET_NAME = {cls_._name: cls_ for cls_ in SpreadsheetListError.__subclasses__()}
|
|
85
|
+
|
|
86
|
+
# Efficient way to set docstring for all classes
|
|
87
|
+
for _cls in ERROR_CLS_BY_SPREADSHEET_NAME.values():
|
|
88
|
+
_cls.__doc__ = SpreadsheetListError.__doc__
|
|
89
|
+
del _cls
|
|
@@ -3,7 +3,7 @@ import xml.etree.ElementTree as ET
|
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
|
|
6
|
-
from ._base import
|
|
6
|
+
from ._base import IssueList, NeatError, NeatWarning
|
|
7
7
|
|
|
8
8
|
__all__ = ["FORMATTER_BY_NAME", "BasicHTML", "Formatter"]
|
|
9
9
|
|
|
@@ -13,14 +13,14 @@ class Formatter(ABC):
|
|
|
13
13
|
default_file_prefix: str = "validation_report"
|
|
14
14
|
|
|
15
15
|
@abstractmethod
|
|
16
|
-
def create_report(self, issues:
|
|
16
|
+
def create_report(self, issues: IssueList) -> str:
|
|
17
17
|
raise NotImplementedError()
|
|
18
18
|
|
|
19
19
|
@property
|
|
20
20
|
def default_file_name(self) -> str:
|
|
21
21
|
return f"{self.default_file_prefix}_{type(self).__name__.lower()}{self.file_suffix}"
|
|
22
22
|
|
|
23
|
-
def write_to_file(self, issues:
|
|
23
|
+
def write_to_file(self, issues: IssueList, file_or_dir_path: Path | None = None) -> None:
|
|
24
24
|
if file_or_dir_path is None:
|
|
25
25
|
file_or_dir_path = Path(self.default_file_name)
|
|
26
26
|
elif file_or_dir_path.is_dir():
|
|
@@ -41,7 +41,7 @@ class BasicHTML(Formatter):
|
|
|
41
41
|
self._doc = ET.Element("html")
|
|
42
42
|
self._body = ET.SubElement(self._doc, "body")
|
|
43
43
|
|
|
44
|
-
def create_report(self, issues:
|
|
44
|
+
def create_report(self, issues: IssueList) -> str:
|
|
45
45
|
errors = [issue for issue in issues if isinstance(issue, NeatError)]
|
|
46
46
|
warnings_ = [issue for issue in issues if isinstance(issue, NeatWarning)]
|
|
47
47
|
self._doc.clear()
|