cognite-neat 0.124.0__py3-none-any.whl → 0.125.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (29) hide show
  1. cognite/neat/_data_model/importers/__init__.py +2 -1
  2. cognite/neat/_data_model/importers/_table_importer/__init__.py +0 -0
  3. cognite/neat/_data_model/importers/_table_importer/data_classes.py +141 -0
  4. cognite/neat/_data_model/importers/_table_importer/importer.py +76 -0
  5. cognite/neat/_data_model/importers/_table_importer/source.py +89 -0
  6. cognite/neat/_exceptions.py +17 -0
  7. cognite/neat/_utils/text.py +22 -0
  8. cognite/neat/_utils/useful_types.py +4 -0
  9. cognite/neat/_utils/validation.py +7 -4
  10. cognite/neat/_version.py +1 -1
  11. cognite/neat/v0/core/_data_model/_constants.py +1 -0
  12. cognite/neat/v0/core/_data_model/exporters/_data_model2excel.py +3 -3
  13. cognite/neat/v0/core/_data_model/importers/_dms2data_model.py +4 -3
  14. cognite/neat/v0/core/_data_model/importers/_spreadsheet2data_model.py +85 -5
  15. cognite/neat/v0/core/_data_model/models/entities/__init__.py +2 -0
  16. cognite/neat/v0/core/_data_model/models/entities/_single_value.py +14 -0
  17. cognite/neat/v0/core/_data_model/models/entities/_types.py +10 -0
  18. cognite/neat/v0/core/_data_model/models/physical/_exporter.py +3 -11
  19. cognite/neat/v0/core/_data_model/models/physical/_unverified.py +61 -12
  20. cognite/neat/v0/core/_data_model/models/physical/_validation.py +8 -4
  21. cognite/neat/v0/core/_data_model/models/physical/_verified.py +86 -15
  22. cognite/neat/v0/core/_data_model/transformers/_converters.py +11 -4
  23. cognite/neat/v0/core/_instances/queries/_select.py +13 -29
  24. cognite/neat/v0/core/_store/_instance.py +2 -2
  25. cognite/neat/v0/core/_utils/spreadsheet.py +17 -3
  26. {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/METADATA +1 -1
  27. {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/RECORD +29 -24
  28. {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/WHEEL +0 -0
  29. {cognite_neat-0.124.0.dist-info → cognite_neat-0.125.1.dist-info}/licenses/LICENSE +0 -0
@@ -1,3 +1,4 @@
1
1
  from ._base import DMSImporter
2
+ from ._table_importer.importer import DMSTableImporter
2
3
 
3
- __all__ = ["DMSImporter"]
4
+ __all__ = ["DMSImporter", "DMSTableImporter"]
@@ -0,0 +1,141 @@
1
+ from collections.abc import Mapping
2
+ from typing import Annotated, cast
3
+
4
+ from pydantic import AliasGenerator, BaseModel, BeforeValidator, Field, model_validator
5
+ from pydantic.alias_generators import to_camel
6
+
7
+ from cognite.neat._data_model.models.entities import ParsedEntity, parse_entities, parse_entity
8
+ from cognite.neat._utils.text import title_case
9
+ from cognite.neat._utils.useful_types import CellValueType
10
+
11
+
12
+ def parse_entity_str(v: str) -> ParsedEntity:
13
+ try:
14
+ return parse_entity(v)
15
+ except ValueError as e:
16
+ raise ValueError(f"Invalid entity syntax: {e}") from e
17
+
18
+
19
+ def parse_entities_str(v: str) -> list[ParsedEntity] | None:
20
+ try:
21
+ return parse_entities(v)
22
+ except ValueError as e:
23
+ raise ValueError(f"Invalid entity list syntax: {e}") from e
24
+
25
+
26
+ Entity = Annotated[ParsedEntity, BeforeValidator(parse_entity_str, str)]
27
+ EntityList = Annotated[list[ParsedEntity], BeforeValidator(parse_entities_str, str)]
28
+
29
+
30
+ class TableObj(
31
+ BaseModel,
32
+ extra="ignore",
33
+ alias_generator=AliasGenerator(
34
+ validation_alias=title_case,
35
+ serialization_alias=to_camel,
36
+ ),
37
+ ): ...
38
+
39
+
40
+ class MetadataValue(TableObj):
41
+ key: str
42
+ value: CellValueType
43
+
44
+
45
+ class DMSProperty(TableObj):
46
+ view: Entity
47
+ view_property: str
48
+ name: str | None = None
49
+ description: str | None = None
50
+ connection: Entity | None
51
+ value_type: Entity
52
+ min_count: int | None
53
+ max_count: int | None
54
+ immutable: bool | None = None
55
+ default: CellValueType | None = None
56
+ auto_increment: bool | None = None
57
+ container: Entity | None = None
58
+ container_property: str | None = None
59
+ container_property_name: str | None = None
60
+ container_property_description: str | None = None
61
+ index: EntityList | None = None
62
+ constraint: EntityList | None = None
63
+
64
+
65
+ class DMSView(TableObj):
66
+ view: Entity
67
+ name: str | None = None
68
+ description: str | None = None
69
+ implements: EntityList | None = None
70
+ filter: str | None = None
71
+ in_model: bool | None = None
72
+
73
+
74
+ class DMSContainer(TableObj):
75
+ container: Entity
76
+ name: str | None = None
77
+ description: str | None = None
78
+ constraint: EntityList | None = None
79
+ used_for: str | None = None
80
+
81
+
82
+ class DMSEnum(TableObj):
83
+ collection: str
84
+ value: str
85
+ name: str | None = None
86
+ description: str | None = None
87
+
88
+
89
+ class DMSNode(TableObj):
90
+ node: Entity
91
+
92
+
93
+ class TableDMS(TableObj):
94
+ metadata: list[MetadataValue]
95
+ properties: list[DMSProperty]
96
+ views: list[DMSView]
97
+ containers: list[DMSContainer] = Field(default_factory=list)
98
+ enum: list[DMSEnum] = Field(default_factory=list)
99
+ nodes: list[DMSNode] = Field(default_factory=list)
100
+
101
+ @model_validator(mode="before")
102
+ def _title_case_keys(
103
+ cls, data: dict[str, list[dict[str, CellValueType]]]
104
+ ) -> dict[str, list[dict[str, CellValueType]]]:
105
+ if isinstance(data, dict):
106
+ # We are case-insensitive on the table names.
107
+ return {title_case(k): v for k, v in data.items()}
108
+ return data
109
+
110
+
111
+ DMS_API_MAPPING: Mapping[str, Mapping[str, str]] = {
112
+ "Views": {
113
+ "space": "View",
114
+ "externalId": "View",
115
+ "version": "View",
116
+ **{
117
+ cast(str, field_.serialization_alias): cast(str, field_.validation_alias)
118
+ for field_id, field_ in DMSView.model_fields.items()
119
+ if field_id != "View"
120
+ },
121
+ },
122
+ "Containers": {
123
+ "space": "Container",
124
+ "externalId": "Container",
125
+ **{
126
+ cast(str, field_.serialization_alias): cast(str, field_.validation_alias)
127
+ for field_id, field_ in DMSContainer.model_fields.items()
128
+ if field_id != "Container"
129
+ },
130
+ },
131
+ "Properties": {
132
+ "space": "View",
133
+ "externalId": "View",
134
+ "property": "ViewProperty",
135
+ **{
136
+ cast(str, field_.serialization_alias): cast(str, field_.validation_alias)
137
+ for field_id, field_ in DMSProperty.model_fields.items()
138
+ if field_id not in ("View", "ViewProperty")
139
+ },
140
+ },
141
+ }
@@ -0,0 +1,76 @@
1
+ from collections.abc import Mapping
2
+ from typing import ClassVar, cast
3
+
4
+ from pydantic import ValidationError
5
+
6
+ from cognite.neat._data_model.importers._base import DMSImporter
7
+ from cognite.neat._data_model.models.dms import (
8
+ RequestSchema,
9
+ )
10
+ from cognite.neat._exceptions import DataModelImportError
11
+ from cognite.neat._issues import ModelSyntaxError
12
+ from cognite.neat._utils.useful_types import CellValueType
13
+ from cognite.neat._utils.validation import as_json_path, humanize_validation_error
14
+
15
+ from .data_classes import TableDMS
16
+
17
+
18
+ class DMSTableImporter(DMSImporter):
19
+ """Imports DMS from a table structure.
20
+
21
+ The tables can are expected to be a dictionary where the keys are the table names and the values
22
+ are lists of dictionaries representing the rows in the table.
23
+ """
24
+
25
+ # We can safely cast as we know the validation_alias is always set to a str.
26
+ REQUIRED_SHEETS = tuple(
27
+ cast(str, field_.validation_alias) for field_ in TableDMS.model_fields.values() if field_.is_required()
28
+ )
29
+ REQUIRED_SHEET_MESSAGES: ClassVar[Mapping[str, str]] = {
30
+ f"Missing required column: {sheet!r}": f"Missing required sheet: {sheet!r}" for sheet in REQUIRED_SHEETS
31
+ }
32
+
33
+ def __init__(self, tables: dict[str, list[dict[str, CellValueType]]]) -> None:
34
+ self._table = tables
35
+
36
+ def to_data_model(self) -> RequestSchema:
37
+ raise NotImplementedError()
38
+
39
+ def _read_tables(self) -> TableDMS:
40
+ try:
41
+ # Check tables, columns, data type and entity syntax.
42
+ table = TableDMS.model_validate(self._table)
43
+ except ValidationError as e:
44
+ errors = self._create_error_messages(e)
45
+ raise DataModelImportError(errors) from None
46
+ return table
47
+
48
+ def _create_error_messages(self, error: ValidationError) -> list[ModelSyntaxError]:
49
+ errors: list[ModelSyntaxError] = []
50
+ seen: set[str] = set()
51
+ for message in humanize_validation_error(
52
+ error,
53
+ humanize_location=self._location,
54
+ field_name="column",
55
+ missing_required_descriptor="missing",
56
+ ):
57
+ # Replace messages about missing required columns with missing required sheets.
58
+ message = self.REQUIRED_SHEET_MESSAGES.get(message, message)
59
+ if message in seen:
60
+ # We treat all rows as the same, so we get duplicated errors for each row.
61
+ continue
62
+ seen.add(message)
63
+ errors.append(ModelSyntaxError(message=message))
64
+ return errors
65
+
66
+ @staticmethod
67
+ def _location(loc: tuple[str | int, ...]) -> str:
68
+ if isinstance(loc[0], str) and len(loc) == 2: # Sheet + row.
69
+ # We skip the row as we treat all rows as the same. For example, if a required column is missing in one
70
+ # row, it is missing in all rows.
71
+ return f"{loc[0]} sheet"
72
+ elif len(loc) == 3 and isinstance(loc[0], str) and isinstance(loc[1], int) and isinstance(loc[2], str):
73
+ # This means there is something wrong in a specific cell.
74
+ return f"{loc[0]} sheet row {loc[1] + 1} column {loc[2]!r}"
75
+ # This should be unreachable as the TableDMS model only has 2 levels.
76
+ return as_json_path(loc)
@@ -0,0 +1,89 @@
1
+ from collections.abc import Mapping
2
+ from dataclasses import dataclass, field
3
+
4
+ from .data_classes import DMS_API_MAPPING
5
+
6
+
7
+ @dataclass
8
+ class SpreadsheetReadContext:
9
+ """This class is used to store information about the source spreadsheet.
10
+
11
+ It is used to adjust row numbers to account for header rows and empty rows
12
+ such that the error/warning messages are accurate.
13
+ """
14
+
15
+ header_row: int = 1
16
+ empty_rows: list[int] = field(default_factory=list)
17
+ skipped_rows: list[int] = field(default_factory=list)
18
+ is_one_indexed: bool = True
19
+
20
+ def __post_init__(self) -> None:
21
+ self.empty_rows.sort()
22
+ self.skipped_rows.sort()
23
+
24
+ def adjusted_row_number(self, row_no: int) -> int:
25
+ output = row_no
26
+ for empty_row in self.empty_rows:
27
+ if empty_row <= output:
28
+ output += 1
29
+ else:
30
+ break
31
+
32
+ for skipped_rows in self.skipped_rows:
33
+ if skipped_rows <= output:
34
+ output += 1
35
+ else:
36
+ break
37
+
38
+ return output + self.header_row + (1 if self.is_one_indexed else 0)
39
+
40
+
41
+ @dataclass
42
+ class TableSource:
43
+ source: str
44
+ table_read: dict[str, SpreadsheetReadContext] = field(default_factory=dict)
45
+
46
+ def location(self, path: tuple[int | str, ...]) -> str:
47
+ table_id: str | None = None
48
+ row_no: int | None = None
49
+ column: str | None = None
50
+ if len(path) >= 1 and isinstance(path[0], str):
51
+ table_id = path[0]
52
+ if len(path) >= 2 and isinstance(path[1], int):
53
+ row_no = path[1]
54
+ if len(path) >= 3 and isinstance(path[2], str):
55
+ column = path[2]
56
+ column = self.field_to_column(table_id, column)
57
+ if isinstance(row_no, int):
58
+ row_no = self.adjust_row_number(table_id, row_no)
59
+ location_parts = []
60
+ if table_id is not None:
61
+ location_parts.append(f"table {table_id!r}")
62
+ if row_no is not None:
63
+ location_parts.append(f"row {row_no}")
64
+ if column is not None:
65
+ location_parts.append(f"column {column!r}")
66
+ if len(path) > 4:
67
+ location_parts.append("-> " + ".".join(str(p) for p in path[3:]))
68
+
69
+ return " ".join(location_parts)
70
+
71
+ def adjust_row_number(self, table_id: str | None, row_no: int) -> int:
72
+ table_read = table_id and self.table_read.get(table_id)
73
+ if table_read:
74
+ return table_read.adjusted_row_number(row_no)
75
+ return row_no + 1 # Convert to 1-indexed if no table read info is available
76
+
77
+ @classmethod
78
+ def field_to_column(cls, table_id: str | None, field_: str) -> str:
79
+ """Maps the field name used in the DMS API to the column named used by Neat."""
80
+ mapping = cls.field_mapping(table_id)
81
+ if mapping is not None:
82
+ return mapping.get(field_, field_)
83
+ return field_
84
+
85
+ @classmethod
86
+ def field_mapping(cls, table_id: str | int | None) -> Mapping[str, str] | None:
87
+ if not isinstance(table_id, str):
88
+ return None
89
+ return DMS_API_MAPPING.get(table_id)
@@ -0,0 +1,17 @@
1
+ from ._issues import ModelSyntaxError
2
+
3
+
4
+ class NeatException(Exception):
5
+ """Base class for all exceptions raised by Neat."""
6
+
7
+ pass
8
+
9
+
10
+ class DataModelImportError(NeatException):
11
+ """Raised when there is an error importing a model."""
12
+
13
+ def __init__(self, errors: list[ModelSyntaxError]) -> None:
14
+ self.errors = errors
15
+
16
+ def __str__(self) -> str:
17
+ return f"Model import failed with {len(self.errors)} errors: " + "; ".join(map(str, self.errors))
@@ -38,3 +38,25 @@ def humanize_collection(collection: Collection[Any], /, *, sort: bool = True, bi
38
38
  sequence = list(strings)
39
39
 
40
40
  return f"{', '.join(sequence[:-1])} {bind_word} {sequence[-1]}"
41
+
42
+
43
+ def title_case(s: str) -> str:
44
+ """Convert a string to title case, handling underscores and hyphens.
45
+
46
+ Args:
47
+ s: The string to convert.
48
+ Returns:
49
+ The title-cased string.
50
+ Examples:
51
+ >>> title_case("hello world")
52
+ 'Hello World'
53
+ >>> title_case("hello_world")
54
+ 'Hello World'
55
+ >>> title_case("hello-world")
56
+ 'Hello World'
57
+ >>> title_case("hello_world-and-universe")
58
+ 'Hello World And Universe'
59
+ >>> title_case("HELLO WORLD")
60
+ 'Hello World'
61
+ """
62
+ return " ".join(word.capitalize() for word in s.replace("_", " ").replace("-", " ").split())
@@ -1,7 +1,11 @@
1
1
  from collections.abc import Hashable
2
+ from datetime import date, datetime, time, timedelta
2
3
  from typing import TypeAlias, TypeVar
3
4
 
4
5
  JsonVal: TypeAlias = None | str | int | float | bool | dict[str, "JsonVal"] | list["JsonVal"]
5
6
 
6
7
 
7
8
  T_ID = TypeVar("T_ID", bound=Hashable)
9
+
10
+ # These are the types that openpyxl supports in cells
11
+ CellValueType: TypeAlias = str | int | float | bool | datetime | date | time | timedelta | None
@@ -31,6 +31,7 @@ def humanize_validation_error(
31
31
  humanize_location: Callable[[tuple[int | str, ...]], str] = as_json_path,
32
32
  field_name: Literal["field", "column", "value"] = "field",
33
33
  field_renaming: Mapping[str, str] | None = None,
34
+ missing_required_descriptor: Literal["empty", "missing"] = "missing",
34
35
  ) -> list[str]:
35
36
  """Converts a ValidationError to a human-readable format.
36
37
 
@@ -50,6 +51,8 @@ def humanize_validation_error(
50
51
  This is useful when the field names in the model are different from the names in the source.
51
52
  For example, if the model field is "asset_id" but the source column is "Asset ID",
52
53
  you can provide a mapping {"asset_id": "Asset ID"} to have the error messages use the source names.
54
+ missing_required_descriptor: How to describe missing required fields. Default is "missing".
55
+ Other option is "empty" which can be more suitable for table data.
53
56
  Returns:
54
57
  A list of human-readable error messages.
55
58
  """
@@ -60,9 +63,9 @@ def humanize_validation_error(
60
63
  loc = (*parent_loc, *item["loc"])
61
64
  error_type = item["type"]
62
65
  if error_type == "missing":
63
- msg = f"Missing required field: {loc[-1]!r}"
66
+ msg = f"Missing required {field_name}: {loc[-1]!r}"
64
67
  elif error_type == "extra_forbidden":
65
- msg = f"Unused field: {loc[-1]!r}"
68
+ msg = f"Unused {field_name}: {loc[-1]!r}"
66
69
  elif error_type == "value_error":
67
70
  msg = str(item["ctx"]["error"])
68
71
  elif error_type == "literal_error":
@@ -92,10 +95,10 @@ def humanize_validation_error(
92
95
 
93
96
  error_suffix = f"{msg[:1].casefold()}{msg[1:]}"
94
97
  if len(loc) > 1 and error_type in {"extra_forbidden", "missing"}:
95
- if field_name == "column":
98
+ if missing_required_descriptor == "empty" and error_type == "missing":
96
99
  # This is a table so we modify the error message.
97
100
  msg = (
98
- f"In {humanize_location(loc[:-1])} the column {field_renaming.get(str(loc[-1]), loc[-1])!r} "
101
+ f"In {humanize_location(loc[:-1])} the {field_name} {field_renaming.get(str(loc[-1]), loc[-1])!r} "
99
102
  "cannot be empty."
100
103
  )
101
104
  else:
cognite/neat/_version.py CHANGED
@@ -1,2 +1,2 @@
1
- __version__ = "0.124.0"
1
+ __version__ = "0.125.1"
2
2
  __engine__ = "^2.0.4"
@@ -43,6 +43,7 @@ class EntityTypes(StrEnum):
43
43
  prefix = "prefix"
44
44
  space = "space"
45
45
  container_index = "container_index"
46
+ container_constraint = "container_constraint"
46
47
  concept_restriction = "conceptRestriction"
47
48
  value_constraint = "valueConstraint"
48
49
  cardinality_constraint = "cardinalityConstraint"
@@ -32,7 +32,7 @@ from cognite.neat.v0.core._data_model.models.data_types import (
32
32
  )
33
33
  from cognite.neat.v0.core._data_model.models.physical._verified import PhysicalDataModel
34
34
  from cognite.neat.v0.core._utils.spreadsheet import (
35
- find_column_with_value,
35
+ find_column_and_row_with_value,
36
36
  generate_data_validation,
37
37
  )
38
38
 
@@ -217,7 +217,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
217
217
  continue
218
218
  ws = workbook[sheet]
219
219
  for col in get_internal_properties():
220
- column_letter = find_column_with_value(ws, col)
220
+ column_letter = find_column_and_row_with_value(ws, col)[0]
221
221
  if column_letter:
222
222
  ws.column_dimensions[column_letter].hidden = True
223
223
 
@@ -451,7 +451,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
451
451
  workbook[sheet_name].add_data_validation(data_validators[data_validator_name])
452
452
 
453
453
  # APPLY VALIDATOR TO SPECIFIC COLUMN
454
- if column_letter := find_column_with_value(workbook[sheet_name], column_name):
454
+ if column_letter := find_column_and_row_with_value(workbook[sheet_name], column_name)[0]:
455
455
  data_validators[data_validator_name].add(f"{column_letter}{3}:{column_letter}{3 + total_rows}")
456
456
 
457
457
  def _create_sheet_with_header(
@@ -54,6 +54,7 @@ from cognite.neat.v0.core._data_model.models.entities import (
54
54
  ReverseConnectionEntity,
55
55
  ViewEntity,
56
56
  )
57
+ from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity
57
58
  from cognite.neat.v0.core._data_model.models.physical import (
58
59
  UnverifiedPhysicalContainer,
59
60
  UnverifiedPhysicalEnum,
@@ -571,17 +572,17 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
571
572
  index.append(ContainerIndexEntity(prefix="inverted", suffix=index_name, order=order))
572
573
  return index or None
573
574
 
574
- def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
575
+ def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[ContainerConstraintEntity] | None:
575
576
  if not isinstance(prop, dm.MappedPropertyApply):
576
577
  return None
577
578
  container = self._all_containers_by_id[prop.container]
578
- unique_constraints: list[str] = []
579
+ unique_constraints: list[ContainerConstraintEntity] = []
579
580
  for constraint_name, constraint_obj in (container.constraints or {}).items():
580
581
  if isinstance(constraint_obj, dm.RequiresConstraint):
581
582
  # This is handled in the .from_container method of DMSContainer
582
583
  continue
583
584
  elif isinstance(constraint_obj, dm.UniquenessConstraint) and prop_id in constraint_obj.properties:
584
- unique_constraints.append(constraint_name)
585
+ unique_constraints.append(ContainerConstraintEntity(prefix="uniqueness", suffix=constraint_name))
585
586
  elif isinstance(constraint_obj, dm.UniquenessConstraint):
586
587
  # This does not apply to this property
587
588
  continue
@@ -10,8 +10,10 @@ import pandas as pd
10
10
  from openpyxl import load_workbook
11
11
  from openpyxl.worksheet.worksheet import Worksheet
12
12
  from pandas import ExcelFile
13
+ from pydantic import ValidationError
13
14
  from rdflib import Namespace, URIRef
14
15
 
16
+ from cognite.neat.v0.core._data_model._constants import SPLIT_ON_COMMA_PATTERN
15
17
  from cognite.neat.v0.core._data_model._shared import (
16
18
  ImportedDataModel,
17
19
  T_UnverifiedDataModel,
@@ -23,6 +25,7 @@ from cognite.neat.v0.core._data_model.models import (
23
25
  SchemaCompleteness,
24
26
  )
25
27
  from cognite.neat.v0.core._data_model.models._import_contexts import SpreadsheetContext
28
+ from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity, ContainerEntity
26
29
  from cognite.neat.v0.core._issues import IssueList, MultiValueError
27
30
  from cognite.neat.v0.core._issues.errors import (
28
31
  FileMissingRequiredFieldError,
@@ -30,7 +33,11 @@ from cognite.neat.v0.core._issues.errors import (
30
33
  FileReadError,
31
34
  )
32
35
  from cognite.neat.v0.core._issues.warnings import FileMissingRequiredFieldWarning
33
- from cognite.neat.v0.core._utils.spreadsheet import SpreadsheetRead, read_individual_sheet
36
+ from cognite.neat.v0.core._utils.spreadsheet import (
37
+ SpreadsheetRead,
38
+ find_column_and_row_with_value,
39
+ read_individual_sheet,
40
+ )
34
41
  from cognite.neat.v0.core._utils.text import humanize_collection
35
42
 
36
43
  from ._base import BaseImporter
@@ -306,7 +313,7 @@ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
306
313
 
307
314
  """
308
315
 
309
- workbook = load_workbook(filepath)
316
+ workbook = load_workbook(filepath, data_only=True)
310
317
 
311
318
  if "Classes" in workbook.sheetnames:
312
319
  print(
@@ -323,13 +330,17 @@ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
323
330
  if "Properties" in workbook.sheetnames:
324
331
  _replace_class_with_concept_cell(workbook["Properties"])
325
332
 
326
- with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
327
- workbook.save(temp_file.name)
328
- return Path(temp_file.name)
333
+ elif "Containers" in workbook.sheetnames:
334
+ _replace_legacy_constraint_form(workbook["Containers"])
335
+ _replace_legacy_constraint_form(workbook["Properties"])
329
336
 
330
337
  else:
331
338
  return filepath
332
339
 
340
+ with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
341
+ workbook.save(temp_file.name)
342
+ return Path(temp_file.name)
343
+
333
344
 
334
345
  def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
335
346
  """
@@ -342,3 +353,72 @@ def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
342
353
  for cell in row:
343
354
  if cell.value == "Class":
344
355
  cell.value = "Concept"
356
+
357
+
358
+ def _replace_legacy_constraint_form(sheet: Worksheet) -> None:
359
+ """
360
+ Replaces the legacy form of container constraints with the new form in the given sheet.
361
+
362
+ Args:
363
+ sheet (Worksheet): The sheet in which to replace the old form of container constraints.
364
+ """
365
+ column, row = find_column_and_row_with_value(sheet, "Constraint", False)
366
+
367
+ if not column or not row:
368
+ return None
369
+
370
+ # Iterate over values in the constraint column and replace old form with new form
371
+ replaced: bool = False
372
+ for cell_row in sheet.iter_rows(min_row=row + 1, min_col=column, max_col=column):
373
+ cell = cell_row[0]
374
+ if cell.value is not None: # Skip empty cells
375
+ # Container sheet update
376
+ if sheet.title.lower() == "containers":
377
+ constraints = []
378
+ for constraint in SPLIT_ON_COMMA_PATTERN.split(str(cell.value)):
379
+ # latest format, do nothing
380
+ if "container" in constraint.lower():
381
+ constraints.append(constraint)
382
+ continue
383
+
384
+ try:
385
+ container = ContainerEntity.load(constraint, space="default")
386
+ container_str = container.external_id if container.space == "default" else str(container)
387
+ constraints.append(
388
+ f"requires:{container.space}_{container.external_id}(container={container_str})"
389
+ )
390
+ replaced = True
391
+ except ValidationError:
392
+ constraints.append(constraint)
393
+
394
+ cell.value = ",".join(constraints)
395
+
396
+ # Properties sheet update
397
+ elif sheet.title.lower() == "properties":
398
+ constraints = []
399
+ for constraint in SPLIT_ON_COMMA_PATTERN.split(str(cell.value)):
400
+ try:
401
+ constraint_entity = ContainerConstraintEntity.load(constraint)
402
+
403
+ if constraint_entity.prefix in ["uniqueness", "requires"]:
404
+ constraints.append(constraint)
405
+
406
+ # Replace old format with new format
407
+ else:
408
+ constraints.append(f"uniqueness:{constraint}")
409
+ replaced = True
410
+
411
+ # If the constraint is not valid, we keep it as is
412
+ # to be caught by validation later
413
+ except ValidationError:
414
+ constraints.append(constraint)
415
+
416
+ cell.value = ",".join(constraints)
417
+
418
+ if replaced:
419
+ print(
420
+ (
421
+ "You are using a legacy container constraints format "
422
+ f"in the {sheet.title} sheet. Please update to the latest format."
423
+ ),
424
+ )
@@ -7,6 +7,7 @@ from ._single_value import (
7
7
  AssetFields,
8
8
  ConceptEntity,
9
9
  ConceptualEntity,
10
+ ContainerConstraintEntity,
10
11
  ContainerEntity,
11
12
  ContainerIndexEntity,
12
13
  DataModelEntity,
@@ -35,6 +36,7 @@ __all__ = [
35
36
  "ConceptPropertyCardinalityConstraint",
36
37
  "ConceptPropertyValueConstraint",
37
38
  "ConceptualEntity",
39
+ "ContainerConstraintEntity",
38
40
  "ContainerEntity",
39
41
  "ContainerEntityList",
40
42
  "ContainerIndexEntity",
@@ -660,3 +660,17 @@ class ContainerIndexEntity(PhysicalEntity[None]):
660
660
  @classmethod
661
661
  def from_id(cls, id: None) -> Self:
662
662
  return cls(suffix="dummy")
663
+
664
+
665
+ class ContainerConstraintEntity(PhysicalEntity[None]):
666
+ type_: ClassVar[EntityTypes] = EntityTypes.container_constraint
667
+ prefix: _UndefinedType | Literal["uniqueness", "requires"] = Undefined # type: ignore[assignment]
668
+ suffix: str
669
+ container: ContainerEntity | None = None
670
+
671
+ def as_id(self) -> None:
672
+ return None
673
+
674
+ @classmethod
675
+ def from_id(cls, id: None) -> Self:
676
+ return cls(suffix="dummy")