cognite-neat 0.124.0__py3-none-any.whl → 0.125.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_data_model/importers/__init__.py +2 -1
- cognite/neat/_data_model/importers/_table_importer/__init__.py +0 -0
- cognite/neat/_data_model/importers/_table_importer/data_classes.py +141 -0
- cognite/neat/_data_model/importers/_table_importer/importer.py +76 -0
- cognite/neat/_data_model/importers/_table_importer/source.py +89 -0
- cognite/neat/_exceptions.py +17 -0
- cognite/neat/_utils/text.py +22 -0
- cognite/neat/_utils/useful_types.py +4 -0
- cognite/neat/_utils/validation.py +7 -4
- cognite/neat/_version.py +1 -1
- cognite/neat/v0/core/_data_model/_constants.py +1 -0
- cognite/neat/v0/core/_data_model/exporters/_data_model2excel.py +3 -3
- cognite/neat/v0/core/_data_model/importers/_dms2data_model.py +4 -3
- cognite/neat/v0/core/_data_model/importers/_spreadsheet2data_model.py +85 -5
- cognite/neat/v0/core/_data_model/models/entities/__init__.py +2 -0
- cognite/neat/v0/core/_data_model/models/entities/_single_value.py +14 -0
- cognite/neat/v0/core/_data_model/models/entities/_types.py +10 -0
- cognite/neat/v0/core/_data_model/models/physical/_exporter.py +3 -11
- cognite/neat/v0/core/_data_model/models/physical/_unverified.py +61 -12
- cognite/neat/v0/core/_data_model/models/physical/_validation.py +8 -4
- cognite/neat/v0/core/_data_model/models/physical/_verified.py +86 -15
- cognite/neat/v0/core/_data_model/transformers/_converters.py +11 -4
- cognite/neat/v0/core/_instances/queries/_select.py +13 -29
- cognite/neat/v0/core/_store/_instance.py +2 -2
- cognite/neat/v0/core/_utils/spreadsheet.py +17 -3
- {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/METADATA +1 -1
- {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/RECORD +29 -24
- {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/WHEEL +0 -0
- {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/licenses/LICENSE +0 -0
|
File without changes
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
from typing import Annotated, cast
|
|
3
|
+
|
|
4
|
+
from pydantic import AliasGenerator, BaseModel, BeforeValidator, Field, model_validator
|
|
5
|
+
from pydantic.alias_generators import to_camel
|
|
6
|
+
|
|
7
|
+
from cognite.neat._data_model.models.entities import ParsedEntity, parse_entities, parse_entity
|
|
8
|
+
from cognite.neat._utils.text import title_case
|
|
9
|
+
from cognite.neat._utils.useful_types import CellValueType
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_entity_str(v: str) -> ParsedEntity:
|
|
13
|
+
try:
|
|
14
|
+
return parse_entity(v)
|
|
15
|
+
except ValueError as e:
|
|
16
|
+
raise ValueError(f"Invalid entity syntax: {e}") from e
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def parse_entities_str(v: str) -> list[ParsedEntity] | None:
|
|
20
|
+
try:
|
|
21
|
+
return parse_entities(v)
|
|
22
|
+
except ValueError as e:
|
|
23
|
+
raise ValueError(f"Invalid entity list syntax: {e}") from e
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
Entity = Annotated[ParsedEntity, BeforeValidator(parse_entity_str, str)]
|
|
27
|
+
EntityList = Annotated[list[ParsedEntity], BeforeValidator(parse_entities_str, str)]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TableObj(
|
|
31
|
+
BaseModel,
|
|
32
|
+
extra="ignore",
|
|
33
|
+
alias_generator=AliasGenerator(
|
|
34
|
+
validation_alias=title_case,
|
|
35
|
+
serialization_alias=to_camel,
|
|
36
|
+
),
|
|
37
|
+
): ...
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class MetadataValue(TableObj):
|
|
41
|
+
key: str
|
|
42
|
+
value: CellValueType
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class DMSProperty(TableObj):
|
|
46
|
+
view: Entity
|
|
47
|
+
view_property: str
|
|
48
|
+
name: str | None = None
|
|
49
|
+
description: str | None = None
|
|
50
|
+
connection: Entity | None
|
|
51
|
+
value_type: Entity
|
|
52
|
+
min_count: int | None
|
|
53
|
+
max_count: int | None
|
|
54
|
+
immutable: bool | None = None
|
|
55
|
+
default: CellValueType | None = None
|
|
56
|
+
auto_increment: bool | None = None
|
|
57
|
+
container: Entity | None = None
|
|
58
|
+
container_property: str | None = None
|
|
59
|
+
container_property_name: str | None = None
|
|
60
|
+
container_property_description: str | None = None
|
|
61
|
+
index: EntityList | None = None
|
|
62
|
+
constraint: EntityList | None = None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class DMSView(TableObj):
|
|
66
|
+
view: Entity
|
|
67
|
+
name: str | None = None
|
|
68
|
+
description: str | None = None
|
|
69
|
+
implements: EntityList | None = None
|
|
70
|
+
filter: str | None = None
|
|
71
|
+
in_model: bool | None = None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class DMSContainer(TableObj):
|
|
75
|
+
container: Entity
|
|
76
|
+
name: str | None = None
|
|
77
|
+
description: str | None = None
|
|
78
|
+
constraint: EntityList | None = None
|
|
79
|
+
used_for: str | None = None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class DMSEnum(TableObj):
|
|
83
|
+
collection: str
|
|
84
|
+
value: str
|
|
85
|
+
name: str | None = None
|
|
86
|
+
description: str | None = None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class DMSNode(TableObj):
|
|
90
|
+
node: Entity
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class TableDMS(TableObj):
|
|
94
|
+
metadata: list[MetadataValue]
|
|
95
|
+
properties: list[DMSProperty]
|
|
96
|
+
views: list[DMSView]
|
|
97
|
+
containers: list[DMSContainer] = Field(default_factory=list)
|
|
98
|
+
enum: list[DMSEnum] = Field(default_factory=list)
|
|
99
|
+
nodes: list[DMSNode] = Field(default_factory=list)
|
|
100
|
+
|
|
101
|
+
@model_validator(mode="before")
|
|
102
|
+
def _title_case_keys(
|
|
103
|
+
cls, data: dict[str, list[dict[str, CellValueType]]]
|
|
104
|
+
) -> dict[str, list[dict[str, CellValueType]]]:
|
|
105
|
+
if isinstance(data, dict):
|
|
106
|
+
# We are case-insensitive on the table names.
|
|
107
|
+
return {title_case(k): v for k, v in data.items()}
|
|
108
|
+
return data
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
DMS_API_MAPPING: Mapping[str, Mapping[str, str]] = {
|
|
112
|
+
"Views": {
|
|
113
|
+
"space": "View",
|
|
114
|
+
"externalId": "View",
|
|
115
|
+
"version": "View",
|
|
116
|
+
**{
|
|
117
|
+
cast(str, field_.serialization_alias): cast(str, field_.validation_alias)
|
|
118
|
+
for field_id, field_ in DMSView.model_fields.items()
|
|
119
|
+
if field_id != "View"
|
|
120
|
+
},
|
|
121
|
+
},
|
|
122
|
+
"Containers": {
|
|
123
|
+
"space": "Container",
|
|
124
|
+
"externalId": "Container",
|
|
125
|
+
**{
|
|
126
|
+
cast(str, field_.serialization_alias): cast(str, field_.validation_alias)
|
|
127
|
+
for field_id, field_ in DMSContainer.model_fields.items()
|
|
128
|
+
if field_id != "Container"
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
"Properties": {
|
|
132
|
+
"space": "View",
|
|
133
|
+
"externalId": "View",
|
|
134
|
+
"property": "ViewProperty",
|
|
135
|
+
**{
|
|
136
|
+
cast(str, field_.serialization_alias): cast(str, field_.validation_alias)
|
|
137
|
+
for field_id, field_ in DMSProperty.model_fields.items()
|
|
138
|
+
if field_id not in ("View", "ViewProperty")
|
|
139
|
+
},
|
|
140
|
+
},
|
|
141
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
from typing import ClassVar, cast
|
|
3
|
+
|
|
4
|
+
from pydantic import ValidationError
|
|
5
|
+
|
|
6
|
+
from cognite.neat._data_model.importers._base import DMSImporter
|
|
7
|
+
from cognite.neat._data_model.models.dms import (
|
|
8
|
+
RequestSchema,
|
|
9
|
+
)
|
|
10
|
+
from cognite.neat._exceptions import DataModelImportError
|
|
11
|
+
from cognite.neat._issues import ModelSyntaxError
|
|
12
|
+
from cognite.neat._utils.useful_types import CellValueType
|
|
13
|
+
from cognite.neat._utils.validation import as_json_path, humanize_validation_error
|
|
14
|
+
|
|
15
|
+
from .data_classes import TableDMS
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DMSTableImporter(DMSImporter):
|
|
19
|
+
"""Imports DMS from a table structure.
|
|
20
|
+
|
|
21
|
+
The tables can are expected to be a dictionary where the keys are the table names and the values
|
|
22
|
+
are lists of dictionaries representing the rows in the table.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
# We can safely cast as we know the validation_alias is always set to a str.
|
|
26
|
+
REQUIRED_SHEETS = tuple(
|
|
27
|
+
cast(str, field_.validation_alias) for field_ in TableDMS.model_fields.values() if field_.is_required()
|
|
28
|
+
)
|
|
29
|
+
REQUIRED_SHEET_MESSAGES: ClassVar[Mapping[str, str]] = {
|
|
30
|
+
f"Missing required column: {sheet!r}": f"Missing required sheet: {sheet!r}" for sheet in REQUIRED_SHEETS
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
def __init__(self, tables: dict[str, list[dict[str, CellValueType]]]) -> None:
|
|
34
|
+
self._table = tables
|
|
35
|
+
|
|
36
|
+
def to_data_model(self) -> RequestSchema:
|
|
37
|
+
raise NotImplementedError()
|
|
38
|
+
|
|
39
|
+
def _read_tables(self) -> TableDMS:
|
|
40
|
+
try:
|
|
41
|
+
# Check tables, columns, data type and entity syntax.
|
|
42
|
+
table = TableDMS.model_validate(self._table)
|
|
43
|
+
except ValidationError as e:
|
|
44
|
+
errors = self._create_error_messages(e)
|
|
45
|
+
raise DataModelImportError(errors) from None
|
|
46
|
+
return table
|
|
47
|
+
|
|
48
|
+
def _create_error_messages(self, error: ValidationError) -> list[ModelSyntaxError]:
|
|
49
|
+
errors: list[ModelSyntaxError] = []
|
|
50
|
+
seen: set[str] = set()
|
|
51
|
+
for message in humanize_validation_error(
|
|
52
|
+
error,
|
|
53
|
+
humanize_location=self._location,
|
|
54
|
+
field_name="column",
|
|
55
|
+
missing_required_descriptor="missing",
|
|
56
|
+
):
|
|
57
|
+
# Replace messages about missing required columns with missing required sheets.
|
|
58
|
+
message = self.REQUIRED_SHEET_MESSAGES.get(message, message)
|
|
59
|
+
if message in seen:
|
|
60
|
+
# We treat all rows as the same, so we get duplicated errors for each row.
|
|
61
|
+
continue
|
|
62
|
+
seen.add(message)
|
|
63
|
+
errors.append(ModelSyntaxError(message=message))
|
|
64
|
+
return errors
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def _location(loc: tuple[str | int, ...]) -> str:
|
|
68
|
+
if isinstance(loc[0], str) and len(loc) == 2: # Sheet + row.
|
|
69
|
+
# We skip the row as we treat all rows as the same. For example, if a required column is missing in one
|
|
70
|
+
# row, it is missing in all rows.
|
|
71
|
+
return f"{loc[0]} sheet"
|
|
72
|
+
elif len(loc) == 3 and isinstance(loc[0], str) and isinstance(loc[1], int) and isinstance(loc[2], str):
|
|
73
|
+
# This means there is something wrong in a specific cell.
|
|
74
|
+
return f"{loc[0]} sheet row {loc[1] + 1} column {loc[2]!r}"
|
|
75
|
+
# This should be unreachable as the TableDMS model only has 2 levels.
|
|
76
|
+
return as_json_path(loc)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
|
|
4
|
+
from .data_classes import DMS_API_MAPPING
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class SpreadsheetReadContext:
|
|
9
|
+
"""This class is used to store information about the source spreadsheet.
|
|
10
|
+
|
|
11
|
+
It is used to adjust row numbers to account for header rows and empty rows
|
|
12
|
+
such that the error/warning messages are accurate.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
header_row: int = 1
|
|
16
|
+
empty_rows: list[int] = field(default_factory=list)
|
|
17
|
+
skipped_rows: list[int] = field(default_factory=list)
|
|
18
|
+
is_one_indexed: bool = True
|
|
19
|
+
|
|
20
|
+
def __post_init__(self) -> None:
|
|
21
|
+
self.empty_rows.sort()
|
|
22
|
+
self.skipped_rows.sort()
|
|
23
|
+
|
|
24
|
+
def adjusted_row_number(self, row_no: int) -> int:
|
|
25
|
+
output = row_no
|
|
26
|
+
for empty_row in self.empty_rows:
|
|
27
|
+
if empty_row <= output:
|
|
28
|
+
output += 1
|
|
29
|
+
else:
|
|
30
|
+
break
|
|
31
|
+
|
|
32
|
+
for skipped_rows in self.skipped_rows:
|
|
33
|
+
if skipped_rows <= output:
|
|
34
|
+
output += 1
|
|
35
|
+
else:
|
|
36
|
+
break
|
|
37
|
+
|
|
38
|
+
return output + self.header_row + (1 if self.is_one_indexed else 0)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class TableSource:
|
|
43
|
+
source: str
|
|
44
|
+
table_read: dict[str, SpreadsheetReadContext] = field(default_factory=dict)
|
|
45
|
+
|
|
46
|
+
def location(self, path: tuple[int | str, ...]) -> str:
|
|
47
|
+
table_id: str | None = None
|
|
48
|
+
row_no: int | None = None
|
|
49
|
+
column: str | None = None
|
|
50
|
+
if len(path) >= 1 and isinstance(path[0], str):
|
|
51
|
+
table_id = path[0]
|
|
52
|
+
if len(path) >= 2 and isinstance(path[1], int):
|
|
53
|
+
row_no = path[1]
|
|
54
|
+
if len(path) >= 3 and isinstance(path[2], str):
|
|
55
|
+
column = path[2]
|
|
56
|
+
column = self.field_to_column(table_id, column)
|
|
57
|
+
if isinstance(row_no, int):
|
|
58
|
+
row_no = self.adjust_row_number(table_id, row_no)
|
|
59
|
+
location_parts = []
|
|
60
|
+
if table_id is not None:
|
|
61
|
+
location_parts.append(f"table {table_id!r}")
|
|
62
|
+
if row_no is not None:
|
|
63
|
+
location_parts.append(f"row {row_no}")
|
|
64
|
+
if column is not None:
|
|
65
|
+
location_parts.append(f"column {column!r}")
|
|
66
|
+
if len(path) > 4:
|
|
67
|
+
location_parts.append("-> " + ".".join(str(p) for p in path[3:]))
|
|
68
|
+
|
|
69
|
+
return " ".join(location_parts)
|
|
70
|
+
|
|
71
|
+
def adjust_row_number(self, table_id: str | None, row_no: int) -> int:
|
|
72
|
+
table_read = table_id and self.table_read.get(table_id)
|
|
73
|
+
if table_read:
|
|
74
|
+
return table_read.adjusted_row_number(row_no)
|
|
75
|
+
return row_no + 1 # Convert to 1-indexed if no table read info is available
|
|
76
|
+
|
|
77
|
+
@classmethod
|
|
78
|
+
def field_to_column(cls, table_id: str | None, field_: str) -> str:
|
|
79
|
+
"""Maps the field name used in the DMS API to the column named used by Neat."""
|
|
80
|
+
mapping = cls.field_mapping(table_id)
|
|
81
|
+
if mapping is not None:
|
|
82
|
+
return mapping.get(field_, field_)
|
|
83
|
+
return field_
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def field_mapping(cls, table_id: str | int | None) -> Mapping[str, str] | None:
|
|
87
|
+
if not isinstance(table_id, str):
|
|
88
|
+
return None
|
|
89
|
+
return DMS_API_MAPPING.get(table_id)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from ._issues import ModelSyntaxError
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class NeatException(Exception):
|
|
5
|
+
"""Base class for all exceptions raised by Neat."""
|
|
6
|
+
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DataModelImportError(NeatException):
|
|
11
|
+
"""Raised when there is an error importing a model."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, errors: list[ModelSyntaxError]) -> None:
|
|
14
|
+
self.errors = errors
|
|
15
|
+
|
|
16
|
+
def __str__(self) -> str:
|
|
17
|
+
return f"Model import failed with {len(self.errors)} errors: " + "; ".join(map(str, self.errors))
|
cognite/neat/_utils/text.py
CHANGED
|
@@ -38,3 +38,25 @@ def humanize_collection(collection: Collection[Any], /, *, sort: bool = True, bi
|
|
|
38
38
|
sequence = list(strings)
|
|
39
39
|
|
|
40
40
|
return f"{', '.join(sequence[:-1])} {bind_word} {sequence[-1]}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def title_case(s: str) -> str:
|
|
44
|
+
"""Convert a string to title case, handling underscores and hyphens.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
s: The string to convert.
|
|
48
|
+
Returns:
|
|
49
|
+
The title-cased string.
|
|
50
|
+
Examples:
|
|
51
|
+
>>> title_case("hello world")
|
|
52
|
+
'Hello World'
|
|
53
|
+
>>> title_case("hello_world")
|
|
54
|
+
'Hello World'
|
|
55
|
+
>>> title_case("hello-world")
|
|
56
|
+
'Hello World'
|
|
57
|
+
>>> title_case("hello_world-and-universe")
|
|
58
|
+
'Hello World And Universe'
|
|
59
|
+
>>> title_case("HELLO WORLD")
|
|
60
|
+
'Hello World'
|
|
61
|
+
"""
|
|
62
|
+
return " ".join(word.capitalize() for word in s.replace("_", " ").replace("-", " ").split())
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
from collections.abc import Hashable
|
|
2
|
+
from datetime import date, datetime, time, timedelta
|
|
2
3
|
from typing import TypeAlias, TypeVar
|
|
3
4
|
|
|
4
5
|
JsonVal: TypeAlias = None | str | int | float | bool | dict[str, "JsonVal"] | list["JsonVal"]
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
T_ID = TypeVar("T_ID", bound=Hashable)
|
|
9
|
+
|
|
10
|
+
# These are the types that openpyxl supports in cells
|
|
11
|
+
CellValueType: TypeAlias = str | int | float | bool | datetime | date | time | timedelta | None
|
|
@@ -31,6 +31,7 @@ def humanize_validation_error(
|
|
|
31
31
|
humanize_location: Callable[[tuple[int | str, ...]], str] = as_json_path,
|
|
32
32
|
field_name: Literal["field", "column", "value"] = "field",
|
|
33
33
|
field_renaming: Mapping[str, str] | None = None,
|
|
34
|
+
missing_required_descriptor: Literal["empty", "missing"] = "missing",
|
|
34
35
|
) -> list[str]:
|
|
35
36
|
"""Converts a ValidationError to a human-readable format.
|
|
36
37
|
|
|
@@ -50,6 +51,8 @@ def humanize_validation_error(
|
|
|
50
51
|
This is useful when the field names in the model are different from the names in the source.
|
|
51
52
|
For example, if the model field is "asset_id" but the source column is "Asset ID",
|
|
52
53
|
you can provide a mapping {"asset_id": "Asset ID"} to have the error messages use the source names.
|
|
54
|
+
missing_required_descriptor: How to describe missing required fields. Default is "missing".
|
|
55
|
+
Other option is "empty" which can be more suitable for table data.
|
|
53
56
|
Returns:
|
|
54
57
|
A list of human-readable error messages.
|
|
55
58
|
"""
|
|
@@ -60,9 +63,9 @@ def humanize_validation_error(
|
|
|
60
63
|
loc = (*parent_loc, *item["loc"])
|
|
61
64
|
error_type = item["type"]
|
|
62
65
|
if error_type == "missing":
|
|
63
|
-
msg = f"Missing required
|
|
66
|
+
msg = f"Missing required {field_name}: {loc[-1]!r}"
|
|
64
67
|
elif error_type == "extra_forbidden":
|
|
65
|
-
msg = f"Unused
|
|
68
|
+
msg = f"Unused {field_name}: {loc[-1]!r}"
|
|
66
69
|
elif error_type == "value_error":
|
|
67
70
|
msg = str(item["ctx"]["error"])
|
|
68
71
|
elif error_type == "literal_error":
|
|
@@ -92,10 +95,10 @@ def humanize_validation_error(
|
|
|
92
95
|
|
|
93
96
|
error_suffix = f"{msg[:1].casefold()}{msg[1:]}"
|
|
94
97
|
if len(loc) > 1 and error_type in {"extra_forbidden", "missing"}:
|
|
95
|
-
if
|
|
98
|
+
if missing_required_descriptor == "empty" and error_type == "missing":
|
|
96
99
|
# This is a table so we modify the error message.
|
|
97
100
|
msg = (
|
|
98
|
-
f"In {humanize_location(loc[:-1])} the
|
|
101
|
+
f"In {humanize_location(loc[:-1])} the {field_name} {field_renaming.get(str(loc[-1]), loc[-1])!r} "
|
|
99
102
|
"cannot be empty."
|
|
100
103
|
)
|
|
101
104
|
else:
|
cognite/neat/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.125.1"
|
|
2
2
|
__engine__ = "^2.0.4"
|
|
@@ -43,6 +43,7 @@ class EntityTypes(StrEnum):
|
|
|
43
43
|
prefix = "prefix"
|
|
44
44
|
space = "space"
|
|
45
45
|
container_index = "container_index"
|
|
46
|
+
container_constraint = "container_constraint"
|
|
46
47
|
concept_restriction = "conceptRestriction"
|
|
47
48
|
value_constraint = "valueConstraint"
|
|
48
49
|
cardinality_constraint = "cardinalityConstraint"
|
|
@@ -32,7 +32,7 @@ from cognite.neat.v0.core._data_model.models.data_types import (
|
|
|
32
32
|
)
|
|
33
33
|
from cognite.neat.v0.core._data_model.models.physical._verified import PhysicalDataModel
|
|
34
34
|
from cognite.neat.v0.core._utils.spreadsheet import (
|
|
35
|
-
|
|
35
|
+
find_column_and_row_with_value,
|
|
36
36
|
generate_data_validation,
|
|
37
37
|
)
|
|
38
38
|
|
|
@@ -217,7 +217,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
|
|
|
217
217
|
continue
|
|
218
218
|
ws = workbook[sheet]
|
|
219
219
|
for col in get_internal_properties():
|
|
220
|
-
column_letter =
|
|
220
|
+
column_letter = find_column_and_row_with_value(ws, col)[0]
|
|
221
221
|
if column_letter:
|
|
222
222
|
ws.column_dimensions[column_letter].hidden = True
|
|
223
223
|
|
|
@@ -451,7 +451,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
|
|
|
451
451
|
workbook[sheet_name].add_data_validation(data_validators[data_validator_name])
|
|
452
452
|
|
|
453
453
|
# APPLY VALIDATOR TO SPECIFIC COLUMN
|
|
454
|
-
if column_letter :=
|
|
454
|
+
if column_letter := find_column_and_row_with_value(workbook[sheet_name], column_name)[0]:
|
|
455
455
|
data_validators[data_validator_name].add(f"{column_letter}{3}:{column_letter}{3 + total_rows}")
|
|
456
456
|
|
|
457
457
|
def _create_sheet_with_header(
|
|
@@ -54,6 +54,7 @@ from cognite.neat.v0.core._data_model.models.entities import (
|
|
|
54
54
|
ReverseConnectionEntity,
|
|
55
55
|
ViewEntity,
|
|
56
56
|
)
|
|
57
|
+
from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity
|
|
57
58
|
from cognite.neat.v0.core._data_model.models.physical import (
|
|
58
59
|
UnverifiedPhysicalContainer,
|
|
59
60
|
UnverifiedPhysicalEnum,
|
|
@@ -571,17 +572,17 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
|
|
|
571
572
|
index.append(ContainerIndexEntity(prefix="inverted", suffix=index_name, order=order))
|
|
572
573
|
return index or None
|
|
573
574
|
|
|
574
|
-
def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[
|
|
575
|
+
def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[ContainerConstraintEntity] | None:
|
|
575
576
|
if not isinstance(prop, dm.MappedPropertyApply):
|
|
576
577
|
return None
|
|
577
578
|
container = self._all_containers_by_id[prop.container]
|
|
578
|
-
unique_constraints: list[
|
|
579
|
+
unique_constraints: list[ContainerConstraintEntity] = []
|
|
579
580
|
for constraint_name, constraint_obj in (container.constraints or {}).items():
|
|
580
581
|
if isinstance(constraint_obj, dm.RequiresConstraint):
|
|
581
582
|
# This is handled in the .from_container method of DMSContainer
|
|
582
583
|
continue
|
|
583
584
|
elif isinstance(constraint_obj, dm.UniquenessConstraint) and prop_id in constraint_obj.properties:
|
|
584
|
-
unique_constraints.append(constraint_name)
|
|
585
|
+
unique_constraints.append(ContainerConstraintEntity(prefix="uniqueness", suffix=constraint_name))
|
|
585
586
|
elif isinstance(constraint_obj, dm.UniquenessConstraint):
|
|
586
587
|
# This does not apply to this property
|
|
587
588
|
continue
|
|
@@ -10,8 +10,10 @@ import pandas as pd
|
|
|
10
10
|
from openpyxl import load_workbook
|
|
11
11
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
12
12
|
from pandas import ExcelFile
|
|
13
|
+
from pydantic import ValidationError
|
|
13
14
|
from rdflib import Namespace, URIRef
|
|
14
15
|
|
|
16
|
+
from cognite.neat.v0.core._data_model._constants import SPLIT_ON_COMMA_PATTERN
|
|
15
17
|
from cognite.neat.v0.core._data_model._shared import (
|
|
16
18
|
ImportedDataModel,
|
|
17
19
|
T_UnverifiedDataModel,
|
|
@@ -23,6 +25,7 @@ from cognite.neat.v0.core._data_model.models import (
|
|
|
23
25
|
SchemaCompleteness,
|
|
24
26
|
)
|
|
25
27
|
from cognite.neat.v0.core._data_model.models._import_contexts import SpreadsheetContext
|
|
28
|
+
from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity, ContainerEntity
|
|
26
29
|
from cognite.neat.v0.core._issues import IssueList, MultiValueError
|
|
27
30
|
from cognite.neat.v0.core._issues.errors import (
|
|
28
31
|
FileMissingRequiredFieldError,
|
|
@@ -30,7 +33,11 @@ from cognite.neat.v0.core._issues.errors import (
|
|
|
30
33
|
FileReadError,
|
|
31
34
|
)
|
|
32
35
|
from cognite.neat.v0.core._issues.warnings import FileMissingRequiredFieldWarning
|
|
33
|
-
from cognite.neat.v0.core._utils.spreadsheet import
|
|
36
|
+
from cognite.neat.v0.core._utils.spreadsheet import (
|
|
37
|
+
SpreadsheetRead,
|
|
38
|
+
find_column_and_row_with_value,
|
|
39
|
+
read_individual_sheet,
|
|
40
|
+
)
|
|
34
41
|
from cognite.neat.v0.core._utils.text import humanize_collection
|
|
35
42
|
|
|
36
43
|
from ._base import BaseImporter
|
|
@@ -306,7 +313,7 @@ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
|
|
|
306
313
|
|
|
307
314
|
"""
|
|
308
315
|
|
|
309
|
-
workbook = load_workbook(filepath)
|
|
316
|
+
workbook = load_workbook(filepath, data_only=True)
|
|
310
317
|
|
|
311
318
|
if "Classes" in workbook.sheetnames:
|
|
312
319
|
print(
|
|
@@ -323,13 +330,17 @@ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
|
|
|
323
330
|
if "Properties" in workbook.sheetnames:
|
|
324
331
|
_replace_class_with_concept_cell(workbook["Properties"])
|
|
325
332
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
333
|
+
elif "Containers" in workbook.sheetnames:
|
|
334
|
+
_replace_legacy_constraint_form(workbook["Containers"])
|
|
335
|
+
_replace_legacy_constraint_form(workbook["Properties"])
|
|
329
336
|
|
|
330
337
|
else:
|
|
331
338
|
return filepath
|
|
332
339
|
|
|
340
|
+
with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
|
|
341
|
+
workbook.save(temp_file.name)
|
|
342
|
+
return Path(temp_file.name)
|
|
343
|
+
|
|
333
344
|
|
|
334
345
|
def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
|
|
335
346
|
"""
|
|
@@ -342,3 +353,72 @@ def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
|
|
|
342
353
|
for cell in row:
|
|
343
354
|
if cell.value == "Class":
|
|
344
355
|
cell.value = "Concept"
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _replace_legacy_constraint_form(sheet: Worksheet) -> None:
|
|
359
|
+
"""
|
|
360
|
+
Replaces the legacy form of container constraints with the new form in the given sheet.
|
|
361
|
+
|
|
362
|
+
Args:
|
|
363
|
+
sheet (Worksheet): The sheet in which to replace the old form of container constraints.
|
|
364
|
+
"""
|
|
365
|
+
column, row = find_column_and_row_with_value(sheet, "Constraint", False)
|
|
366
|
+
|
|
367
|
+
if not column or not row:
|
|
368
|
+
return None
|
|
369
|
+
|
|
370
|
+
# Iterate over values in the constraint column and replace old form with new form
|
|
371
|
+
replaced: bool = False
|
|
372
|
+
for cell_row in sheet.iter_rows(min_row=row + 1, min_col=column, max_col=column):
|
|
373
|
+
cell = cell_row[0]
|
|
374
|
+
if cell.value is not None: # Skip empty cells
|
|
375
|
+
# Container sheet update
|
|
376
|
+
if sheet.title.lower() == "containers":
|
|
377
|
+
constraints = []
|
|
378
|
+
for constraint in SPLIT_ON_COMMA_PATTERN.split(str(cell.value)):
|
|
379
|
+
# latest format, do nothing
|
|
380
|
+
if "container" in constraint.lower():
|
|
381
|
+
constraints.append(constraint)
|
|
382
|
+
continue
|
|
383
|
+
|
|
384
|
+
try:
|
|
385
|
+
container = ContainerEntity.load(constraint, space="default")
|
|
386
|
+
container_str = container.external_id if container.space == "default" else str(container)
|
|
387
|
+
constraints.append(
|
|
388
|
+
f"requires:{container.space}_{container.external_id}(container={container_str})"
|
|
389
|
+
)
|
|
390
|
+
replaced = True
|
|
391
|
+
except ValidationError:
|
|
392
|
+
constraints.append(constraint)
|
|
393
|
+
|
|
394
|
+
cell.value = ",".join(constraints)
|
|
395
|
+
|
|
396
|
+
# Properties sheet update
|
|
397
|
+
elif sheet.title.lower() == "properties":
|
|
398
|
+
constraints = []
|
|
399
|
+
for constraint in SPLIT_ON_COMMA_PATTERN.split(str(cell.value)):
|
|
400
|
+
try:
|
|
401
|
+
constraint_entity = ContainerConstraintEntity.load(constraint)
|
|
402
|
+
|
|
403
|
+
if constraint_entity.prefix in ["uniqueness", "requires"]:
|
|
404
|
+
constraints.append(constraint)
|
|
405
|
+
|
|
406
|
+
# Replace old format with new format
|
|
407
|
+
else:
|
|
408
|
+
constraints.append(f"uniqueness:{constraint}")
|
|
409
|
+
replaced = True
|
|
410
|
+
|
|
411
|
+
# If the constraint is not valid, we keep it as is
|
|
412
|
+
# to be caught by validation later
|
|
413
|
+
except ValidationError:
|
|
414
|
+
constraints.append(constraint)
|
|
415
|
+
|
|
416
|
+
cell.value = ",".join(constraints)
|
|
417
|
+
|
|
418
|
+
if replaced:
|
|
419
|
+
print(
|
|
420
|
+
(
|
|
421
|
+
"You are using a legacy container constraints format "
|
|
422
|
+
f"in the {sheet.title} sheet. Please update to the latest format."
|
|
423
|
+
),
|
|
424
|
+
)
|
|
@@ -7,6 +7,7 @@ from ._single_value import (
|
|
|
7
7
|
AssetFields,
|
|
8
8
|
ConceptEntity,
|
|
9
9
|
ConceptualEntity,
|
|
10
|
+
ContainerConstraintEntity,
|
|
10
11
|
ContainerEntity,
|
|
11
12
|
ContainerIndexEntity,
|
|
12
13
|
DataModelEntity,
|
|
@@ -35,6 +36,7 @@ __all__ = [
|
|
|
35
36
|
"ConceptPropertyCardinalityConstraint",
|
|
36
37
|
"ConceptPropertyValueConstraint",
|
|
37
38
|
"ConceptualEntity",
|
|
39
|
+
"ContainerConstraintEntity",
|
|
38
40
|
"ContainerEntity",
|
|
39
41
|
"ContainerEntityList",
|
|
40
42
|
"ContainerIndexEntity",
|
|
@@ -660,3 +660,17 @@ class ContainerIndexEntity(PhysicalEntity[None]):
|
|
|
660
660
|
@classmethod
|
|
661
661
|
def from_id(cls, id: None) -> Self:
|
|
662
662
|
return cls(suffix="dummy")
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
class ContainerConstraintEntity(PhysicalEntity[None]):
|
|
666
|
+
type_: ClassVar[EntityTypes] = EntityTypes.container_constraint
|
|
667
|
+
prefix: _UndefinedType | Literal["uniqueness", "requires"] = Undefined # type: ignore[assignment]
|
|
668
|
+
suffix: str
|
|
669
|
+
container: ContainerEntity | None = None
|
|
670
|
+
|
|
671
|
+
def as_id(self) -> None:
|
|
672
|
+
return None
|
|
673
|
+
|
|
674
|
+
@classmethod
|
|
675
|
+
def from_id(cls, id: None) -> Self:
|
|
676
|
+
return cls(suffix="dummy")
|