cognite-neat 0.126.0__py3-none-any.whl → 0.126.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/api.py +8 -0
- cognite/neat/_client/client.py +19 -0
- cognite/neat/_client/config.py +40 -0
- cognite/neat/_client/containers_api.py +73 -0
- cognite/neat/_client/data_classes.py +10 -0
- cognite/neat/_client/data_model_api.py +63 -0
- cognite/neat/_client/spaces_api.py +67 -0
- cognite/neat/_client/views_api.py +82 -0
- cognite/neat/_data_model/_analysis.py +127 -0
- cognite/neat/_data_model/_constants.py +59 -0
- cognite/neat/_data_model/_shared.py +46 -0
- cognite/neat/_data_model/deployer/__init__.py +0 -0
- cognite/neat/_data_model/deployer/_differ.py +113 -0
- cognite/neat/_data_model/deployer/_differ_container.py +354 -0
- cognite/neat/_data_model/deployer/_differ_data_model.py +29 -0
- cognite/neat/_data_model/deployer/_differ_space.py +9 -0
- cognite/neat/_data_model/deployer/_differ_view.py +194 -0
- cognite/neat/_data_model/deployer/data_classes.py +176 -0
- cognite/neat/_data_model/exporters/__init__.py +4 -0
- cognite/neat/_data_model/exporters/_base.py +6 -1
- cognite/neat/_data_model/exporters/_table_exporter/__init__.py +0 -0
- cognite/neat/_data_model/exporters/_table_exporter/exporter.py +106 -0
- cognite/neat/_data_model/exporters/_table_exporter/workbook.py +414 -0
- cognite/neat/_data_model/exporters/_table_exporter/writer.py +391 -0
- cognite/neat/_data_model/importers/__init__.py +2 -1
- cognite/neat/_data_model/importers/_api_importer.py +88 -0
- cognite/neat/_data_model/importers/_table_importer/data_classes.py +48 -8
- cognite/neat/_data_model/importers/_table_importer/importer.py +74 -5
- cognite/neat/_data_model/importers/_table_importer/reader.py +63 -7
- cognite/neat/_data_model/models/dms/__init__.py +17 -1
- cognite/neat/_data_model/models/dms/_base.py +12 -8
- cognite/neat/_data_model/models/dms/_constants.py +1 -1
- cognite/neat/_data_model/models/dms/_constraints.py +2 -1
- cognite/neat/_data_model/models/dms/_container.py +5 -5
- cognite/neat/_data_model/models/dms/_data_model.py +3 -3
- cognite/neat/_data_model/models/dms/_data_types.py +8 -1
- cognite/neat/_data_model/models/dms/_http.py +18 -0
- cognite/neat/_data_model/models/dms/_indexes.py +2 -1
- cognite/neat/_data_model/models/dms/_references.py +17 -4
- cognite/neat/_data_model/models/dms/_space.py +11 -7
- cognite/neat/_data_model/models/dms/_view_property.py +7 -4
- cognite/neat/_data_model/models/dms/_views.py +16 -6
- cognite/neat/_data_model/validation/__init__.py +0 -0
- cognite/neat/_data_model/validation/_base.py +16 -0
- cognite/neat/_data_model/validation/dms/__init__.py +9 -0
- cognite/neat/_data_model/validation/dms/_orchestrator.py +68 -0
- cognite/neat/_data_model/validation/dms/_validators.py +139 -0
- cognite/neat/_exceptions.py +15 -3
- cognite/neat/_issues.py +39 -6
- cognite/neat/_session/__init__.py +3 -0
- cognite/neat/_session/_physical.py +88 -0
- cognite/neat/_session/_session.py +34 -25
- cognite/neat/_session/_wrappers.py +61 -0
- cognite/neat/_state_machine/__init__.py +10 -0
- cognite/neat/{_session/_state_machine → _state_machine}/_base.py +11 -1
- cognite/neat/_state_machine/_states.py +53 -0
- cognite/neat/_store/__init__.py +3 -0
- cognite/neat/_store/_provenance.py +55 -0
- cognite/neat/_store/_store.py +124 -0
- cognite/neat/_utils/_reader.py +194 -0
- cognite/neat/_utils/http_client/__init__.py +14 -20
- cognite/neat/_utils/http_client/_client.py +22 -61
- cognite/neat/_utils/http_client/_data_classes.py +167 -268
- cognite/neat/_utils/text.py +6 -0
- cognite/neat/_utils/useful_types.py +23 -2
- cognite/neat/_version.py +1 -1
- cognite/neat/v0/core/_data_model/importers/_rdf/_shared.py +2 -2
- {cognite_neat-0.126.0.dist-info → cognite_neat-0.126.1.dist-info}/METADATA +1 -1
- {cognite_neat-0.126.0.dist-info → cognite_neat-0.126.1.dist-info}/RECORD +72 -38
- cognite/neat/_data_model/exporters/_table_exporter.py +0 -35
- cognite/neat/_session/_state_machine/__init__.py +0 -23
- cognite/neat/_session/_state_machine/_states.py +0 -150
- {cognite_neat-0.126.0.dist-info → cognite_neat-0.126.1.dist-info}/WHEEL +0 -0
- {cognite_neat-0.126.0.dist-info → cognite_neat-0.126.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
from collections.abc import Mapping
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
from typing import ClassVar, cast
|
|
3
4
|
|
|
5
|
+
import yaml
|
|
6
|
+
from openpyxl import load_workbook
|
|
7
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
4
8
|
from pydantic import ValidationError
|
|
5
9
|
|
|
6
10
|
from cognite.neat._data_model.importers._base import DMSImporter
|
|
7
11
|
from cognite.neat._data_model.models.dms import (
|
|
8
12
|
RequestSchema,
|
|
9
13
|
)
|
|
10
|
-
from cognite.neat._exceptions import
|
|
14
|
+
from cognite.neat._exceptions import DataModelImportException
|
|
11
15
|
from cognite.neat._issues import ModelSyntaxError
|
|
12
16
|
from cognite.neat._utils.text import humanize_collection
|
|
13
|
-
from cognite.neat._utils.useful_types import DataModelTableType
|
|
17
|
+
from cognite.neat._utils.useful_types import CellValueType, DataModelTableType
|
|
14
18
|
from cognite.neat._utils.validation import as_json_path, humanize_validation_error
|
|
15
19
|
|
|
16
20
|
from .data_classes import MetadataValue, TableDMS
|
|
17
21
|
from .reader import DMSTableReader
|
|
18
|
-
from .source import TableSource
|
|
22
|
+
from .source import SpreadsheetReadContext, TableSource
|
|
19
23
|
|
|
20
24
|
|
|
21
25
|
class DMSTableImporter(DMSImporter):
|
|
@@ -32,6 +36,7 @@ class DMSTableImporter(DMSImporter):
|
|
|
32
36
|
REQUIRED_SHEET_MESSAGES: ClassVar[Mapping[str, str]] = {
|
|
33
37
|
f"Missing required column: {sheet!r}": f"Missing required sheet: {sheet!r}" for sheet in REQUIRED_SHEETS
|
|
34
38
|
}
|
|
39
|
+
MetadataSheet = cast(str, TableDMS.model_fields["metadata"].validation_alias)
|
|
35
40
|
|
|
36
41
|
def __init__(self, tables: DataModelTableType, source: TableSource | None = None) -> None:
|
|
37
42
|
self._table = tables
|
|
@@ -44,13 +49,40 @@ class DMSTableImporter(DMSImporter):
|
|
|
44
49
|
reader = DMSTableReader(space, version, self._source)
|
|
45
50
|
return reader.read_tables(tables)
|
|
46
51
|
|
|
52
|
+
@classmethod
|
|
53
|
+
def from_yaml(cls, yaml_file: Path) -> "DMSTableImporter":
|
|
54
|
+
"""Create a DMSTableImporter from a YAML file."""
|
|
55
|
+
source = cls._display_name(yaml_file)
|
|
56
|
+
return cls(yaml.safe_load(yaml_file.read_text()), TableSource(source.as_posix()))
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def from_excel(cls, excel_file: Path) -> "DMSTableImporter":
|
|
60
|
+
"""Create a DMSTableImporter from an Excel file."""
|
|
61
|
+
tables: DataModelTableType = {}
|
|
62
|
+
source = TableSource(cls._display_name(excel_file).as_posix())
|
|
63
|
+
workbook = load_workbook(excel_file, read_only=True, data_only=True, rich_text=False)
|
|
64
|
+
try:
|
|
65
|
+
for column_id, column_info in TableDMS.model_fields.items():
|
|
66
|
+
sheet_name = cast(str, column_info.validation_alias)
|
|
67
|
+
if sheet_name not in workbook.sheetnames:
|
|
68
|
+
continue
|
|
69
|
+
required_columns = TableDMS.get_sheet_columns(column_id, column_info, column_type="required")
|
|
70
|
+
sheet = workbook[sheet_name]
|
|
71
|
+
context = SpreadsheetReadContext()
|
|
72
|
+
table_rows = cls._read_rows(sheet, required_columns, context)
|
|
73
|
+
tables[sheet_name] = table_rows
|
|
74
|
+
source.table_read[sheet_name] = context
|
|
75
|
+
return cls(tables, source)
|
|
76
|
+
finally:
|
|
77
|
+
workbook.close()
|
|
78
|
+
|
|
47
79
|
def _read_tables(self) -> TableDMS:
|
|
48
80
|
try:
|
|
49
81
|
# Check tables, columns, data type and entity syntax.
|
|
50
82
|
table = TableDMS.model_validate(self._table)
|
|
51
83
|
except ValidationError as e:
|
|
52
84
|
errors = self._create_error_messages(e)
|
|
53
|
-
raise
|
|
85
|
+
raise DataModelImportException(errors) from None
|
|
54
86
|
return table
|
|
55
87
|
|
|
56
88
|
def _create_error_messages(self, error: ValidationError) -> list[ModelSyntaxError]:
|
|
@@ -99,5 +131,42 @@ class DMSTableImporter(DMSImporter):
|
|
|
99
131
|
if missing:
|
|
100
132
|
error = ModelSyntaxError(message=f"In Metadata missing required values: {humanize_collection(missing)}")
|
|
101
133
|
# If space or version is missing, we cannot continue parsing the model as these are used as defaults.
|
|
102
|
-
raise
|
|
134
|
+
raise DataModelImportException([error]) from None
|
|
103
135
|
return str(default_space), str(default_version)
|
|
136
|
+
|
|
137
|
+
@classmethod
|
|
138
|
+
def _display_name(cls, filepath: Path) -> Path:
|
|
139
|
+
"""Get a display-friendly version of the file path."""
|
|
140
|
+
cwd = Path.cwd()
|
|
141
|
+
source = filepath
|
|
142
|
+
if filepath.is_relative_to(cwd):
|
|
143
|
+
source = filepath.relative_to(cwd)
|
|
144
|
+
return source
|
|
145
|
+
|
|
146
|
+
@classmethod
|
|
147
|
+
def _read_rows(
|
|
148
|
+
cls, sheet: Worksheet, required_columns: list[str], context: SpreadsheetReadContext
|
|
149
|
+
) -> list[dict[str, CellValueType]]:
|
|
150
|
+
table_rows: list[dict[str, CellValueType]] = []
|
|
151
|
+
# Metadata sheet is just a key-value pair of the first two columns.
|
|
152
|
+
# For other sheets, we need to find the column header row first.
|
|
153
|
+
columns: list[str] = [] if sheet.title != cls.MetadataSheet else required_columns
|
|
154
|
+
for row_no, row in enumerate(sheet.iter_rows(values_only=True)):
|
|
155
|
+
if columns:
|
|
156
|
+
# We have found the column header row, read the data rows.
|
|
157
|
+
if all(cell is None for cell in row):
|
|
158
|
+
context.empty_rows.append(row_no)
|
|
159
|
+
else:
|
|
160
|
+
record = dict(zip(columns, row, strict=False))
|
|
161
|
+
# MyPy complains as it thinks DataTableFormula | ArrayFormula could be cell values,
|
|
162
|
+
# but as we used values_only=True, this is not the case.
|
|
163
|
+
table_rows.append(record) # type: ignore[arg-type]
|
|
164
|
+
else:
|
|
165
|
+
# Look for the column header row.
|
|
166
|
+
row_values = [str(cell) for cell in row]
|
|
167
|
+
if set(row_values).intersection(required_columns):
|
|
168
|
+
columns = row_values
|
|
169
|
+
context.header_row = row_no
|
|
170
|
+
else:
|
|
171
|
+
context.skipped_rows.append(row_no)
|
|
172
|
+
return table_rows
|
|
@@ -22,7 +22,7 @@ from cognite.neat._data_model.models.dms import (
|
|
|
22
22
|
ViewRequestPropertyAdapter,
|
|
23
23
|
)
|
|
24
24
|
from cognite.neat._data_model.models.entities import ParsedEntity, parse_entity
|
|
25
|
-
from cognite.neat._exceptions import
|
|
25
|
+
from cognite.neat._exceptions import DataModelImportException
|
|
26
26
|
from cognite.neat._issues import ModelSyntaxError
|
|
27
27
|
from cognite.neat._utils.text import humanize_collection
|
|
28
28
|
from cognite.neat._utils.validation import humanize_validation_error
|
|
@@ -151,6 +151,7 @@ class DMSTableReader:
|
|
|
151
151
|
|
|
152
152
|
class ContainerColumns:
|
|
153
153
|
container = cast(str, DMSContainer.model_fields["container"].validation_alias)
|
|
154
|
+
constraint = cast(str, DMSContainer.model_fields["constraint"].validation_alias)
|
|
154
155
|
|
|
155
156
|
class ViewColumns:
|
|
156
157
|
view = cast(str, DMSView.model_fields["view"].validation_alias)
|
|
@@ -173,7 +174,7 @@ class DMSTableReader:
|
|
|
173
174
|
data_model = self.read_data_model(tables, valid_view_entities)
|
|
174
175
|
|
|
175
176
|
if self.errors:
|
|
176
|
-
raise
|
|
177
|
+
raise DataModelImportException(self.errors) from None
|
|
177
178
|
return RequestSchema(
|
|
178
179
|
dataModel=data_model, views=views, containers=containers, spaces=[space_request], nodeTypes=node_types
|
|
179
180
|
)
|
|
@@ -182,7 +183,7 @@ class DMSTableReader:
|
|
|
182
183
|
space_request = self._validate_obj(SpaceRequest, {"space": space}, (self.Sheets.metadata,), field_name="value")
|
|
183
184
|
if space_request is None:
|
|
184
185
|
# If space is invalid, we stop parsing to avoid raising an error for every place the space is used.
|
|
185
|
-
raise
|
|
186
|
+
raise DataModelImportException(self.errors) from None
|
|
186
187
|
return space_request
|
|
187
188
|
|
|
188
189
|
def read_nodes(self, nodes: list[DMSNode]) -> list[NodeReference]:
|
|
@@ -427,7 +428,7 @@ class DMSTableReader:
|
|
|
427
428
|
return
|
|
428
429
|
loc = (self.Sheets.properties, row_no, self.PropertyColumns.constraint)
|
|
429
430
|
for constraint in prop.constraint:
|
|
430
|
-
data = self.
|
|
431
|
+
data = self.read_property_constraint(constraint, prop.container_property)
|
|
431
432
|
created = self._validate_adapter(ConstraintAdapter, data, loc)
|
|
432
433
|
if created is None:
|
|
433
434
|
continue
|
|
@@ -443,7 +444,7 @@ class DMSTableReader:
|
|
|
443
444
|
)
|
|
444
445
|
|
|
445
446
|
@staticmethod
|
|
446
|
-
def
|
|
447
|
+
def read_property_constraint(constraint: ParsedEntity, prop_id: str) -> dict[str, Any]:
|
|
447
448
|
return {"constraintType": constraint.prefix, "properties": [prop_id], **constraint.properties}
|
|
448
449
|
|
|
449
450
|
def read_view_property(self, prop: DMSProperty, loc: tuple[str | int, ...]) -> dict[str, Any]:
|
|
@@ -574,6 +575,22 @@ class DMSTableReader:
|
|
|
574
575
|
containers_requests: list[ContainerRequest] = []
|
|
575
576
|
rows_by_seen: dict[ParsedEntity, list[int]] = defaultdict(list)
|
|
576
577
|
for row_no, container in enumerate(containers):
|
|
578
|
+
property_constraints = properties.constraints.get(container.container, {})
|
|
579
|
+
require_constraints = self.read_container_constraints(container, row_no)
|
|
580
|
+
if conflict := set(property_constraints.keys()).intersection(set(require_constraints.keys())):
|
|
581
|
+
conflict_str = humanize_collection(conflict)
|
|
582
|
+
location_str = self.source.location((self.Sheets.containers, row_no, self.ContainerColumns.constraint))
|
|
583
|
+
self.errors.append(
|
|
584
|
+
ModelSyntaxError(
|
|
585
|
+
message=(
|
|
586
|
+
f"In {location_str} the container '{container.container!s}' has constraints defined "
|
|
587
|
+
f"with the same identifier(s) as the uniqueness constraint defined in the "
|
|
588
|
+
f"{self.Sheets.properties} sheet. Ensure that the identifiers are unique. "
|
|
589
|
+
f"Conflicting identifiers: {conflict_str}. "
|
|
590
|
+
)
|
|
591
|
+
)
|
|
592
|
+
)
|
|
593
|
+
constraints = {**property_constraints, **require_constraints}
|
|
577
594
|
container_request = self._validate_obj(
|
|
578
595
|
ContainerRequest,
|
|
579
596
|
dict(
|
|
@@ -583,7 +600,7 @@ class DMSTableReader:
|
|
|
583
600
|
description=container.description,
|
|
584
601
|
properties=properties.container[container.container],
|
|
585
602
|
indexes=properties.indices.get(container.container),
|
|
586
|
-
constraints=
|
|
603
|
+
constraints=constraints or None,
|
|
587
604
|
),
|
|
588
605
|
(self.Sheets.containers, row_no),
|
|
589
606
|
)
|
|
@@ -608,6 +625,45 @@ class DMSTableReader:
|
|
|
608
625
|
)
|
|
609
626
|
return containers_requests
|
|
610
627
|
|
|
628
|
+
def read_container_constraints(self, container: DMSContainer, row_no: int) -> dict[str, Constraint]:
|
|
629
|
+
constraints: dict[str, Constraint] = {}
|
|
630
|
+
if not container.constraint:
|
|
631
|
+
return constraints
|
|
632
|
+
for entity in container.constraint:
|
|
633
|
+
loc = self.Sheets.containers, row_no, self.ContainerColumns.constraint
|
|
634
|
+
if entity.prefix != "requires":
|
|
635
|
+
self.errors.append(
|
|
636
|
+
ModelSyntaxError(
|
|
637
|
+
message=(
|
|
638
|
+
f"In {self.source.location(loc)} the constraint '{entity.suffix}' on container "
|
|
639
|
+
f"'{container.container!s}' has an invalid type '{entity.prefix}'. Only 'requires' "
|
|
640
|
+
f"constraints are supported at the container level."
|
|
641
|
+
)
|
|
642
|
+
)
|
|
643
|
+
)
|
|
644
|
+
continue
|
|
645
|
+
|
|
646
|
+
if "require" not in entity.properties:
|
|
647
|
+
self.errors.append(
|
|
648
|
+
ModelSyntaxError(
|
|
649
|
+
message=(
|
|
650
|
+
f"In {self.source.location(loc)} the constraint '{entity.suffix}' on container "
|
|
651
|
+
f"'{container.container!s}' is missing the "
|
|
652
|
+
f"'require' property which is required for container level constraints."
|
|
653
|
+
)
|
|
654
|
+
)
|
|
655
|
+
)
|
|
656
|
+
continue
|
|
657
|
+
data = {
|
|
658
|
+
"constraintType": entity.prefix,
|
|
659
|
+
"require": self._create_container_ref_unparsed(entity.properties["require"], loc),
|
|
660
|
+
}
|
|
661
|
+
created = self._validate_adapter(ConstraintAdapter, data, loc)
|
|
662
|
+
if created is None:
|
|
663
|
+
continue
|
|
664
|
+
constraints[entity.suffix] = created
|
|
665
|
+
return constraints
|
|
666
|
+
|
|
611
667
|
def read_views(
|
|
612
668
|
self,
|
|
613
669
|
views: list[DMSView],
|
|
@@ -674,7 +730,7 @@ class DMSTableReader:
|
|
|
674
730
|
model = self._validate_obj(DataModelRequest, data, (self.Sheets.metadata,), field_name="value")
|
|
675
731
|
if model is None:
|
|
676
732
|
# This is the last step, so we can raise the error here.
|
|
677
|
-
raise
|
|
733
|
+
raise DataModelImportException(self.errors) from None
|
|
678
734
|
return model
|
|
679
735
|
|
|
680
736
|
def _parse_entity(self, entity: str, loc: tuple[str | int, ...]) -> ParsedEntity | None:
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
from cognite.neat._data_model.models.dms._base import
|
|
1
|
+
from cognite.neat._data_model.models.dms._base import (
|
|
2
|
+
APIResource,
|
|
3
|
+
BaseModelObject,
|
|
4
|
+
Resource,
|
|
5
|
+
T_Resource,
|
|
6
|
+
WriteableResource,
|
|
7
|
+
)
|
|
2
8
|
from cognite.neat._data_model.models.dms._constraints import (
|
|
3
9
|
Constraint,
|
|
4
10
|
ConstraintAdapter,
|
|
@@ -13,6 +19,7 @@ from cognite.neat._data_model.models.dms._container import (
|
|
|
13
19
|
ContainerResponse,
|
|
14
20
|
)
|
|
15
21
|
from cognite.neat._data_model.models.dms._data_types import (
|
|
22
|
+
DMS_DATA_TYPES,
|
|
16
23
|
BooleanProperty,
|
|
17
24
|
DataType,
|
|
18
25
|
DataTypeAdapter,
|
|
@@ -38,11 +45,13 @@ from cognite.neat._data_model.models.dms._indexes import BtreeIndex, Index, Inde
|
|
|
38
45
|
from cognite.neat._data_model.models.dms._space import Space, SpaceRequest, SpaceResponse
|
|
39
46
|
|
|
40
47
|
from ._data_model import DataModelRequest, DataModelResponse
|
|
48
|
+
from ._http import DataModelBody, DataModelResource
|
|
41
49
|
from ._references import (
|
|
42
50
|
ContainerDirectReference,
|
|
43
51
|
ContainerReference,
|
|
44
52
|
DataModelReference,
|
|
45
53
|
NodeReference,
|
|
54
|
+
SpaceReference,
|
|
46
55
|
ViewDirectReference,
|
|
47
56
|
ViewReference,
|
|
48
57
|
)
|
|
@@ -70,6 +79,9 @@ from ._views import (
|
|
|
70
79
|
)
|
|
71
80
|
|
|
72
81
|
__all__ = [
|
|
82
|
+
"DMS_DATA_TYPES",
|
|
83
|
+
"APIResource",
|
|
84
|
+
"BaseModelObject",
|
|
73
85
|
"BooleanProperty",
|
|
74
86
|
"BtreeIndex",
|
|
75
87
|
"ConnectionPropertyDefinition",
|
|
@@ -83,8 +95,10 @@ __all__ = [
|
|
|
83
95
|
"ContainerReference",
|
|
84
96
|
"ContainerRequest",
|
|
85
97
|
"ContainerResponse",
|
|
98
|
+
"DataModelBody",
|
|
86
99
|
"DataModelReference",
|
|
87
100
|
"DataModelRequest",
|
|
101
|
+
"DataModelResource",
|
|
88
102
|
"DataModelResponse",
|
|
89
103
|
"DataType",
|
|
90
104
|
"DataTypeAdapter",
|
|
@@ -118,8 +132,10 @@ __all__ = [
|
|
|
118
132
|
"SingleReverseDirectRelationPropertyRequest",
|
|
119
133
|
"SingleReverseDirectRelationPropertyResponse",
|
|
120
134
|
"Space",
|
|
135
|
+
"SpaceReference",
|
|
121
136
|
"SpaceRequest",
|
|
122
137
|
"SpaceResponse",
|
|
138
|
+
"T_Resource",
|
|
123
139
|
"TextProperty",
|
|
124
140
|
"TextProperty",
|
|
125
141
|
"TimeseriesCDFExternalIdReference",
|
|
@@ -1,14 +1,7 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from typing import Generic, TypeVar
|
|
3
3
|
|
|
4
|
-
from
|
|
5
|
-
from pydantic.alias_generators import to_camel
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class BaseModelObject(BaseModel, alias_generator=to_camel, extra="ignore"):
|
|
9
|
-
"""Base class for all object. This includes resources and nested objects."""
|
|
10
|
-
|
|
11
|
-
...
|
|
4
|
+
from cognite.neat._utils.useful_types import BaseModelObject, T_Reference
|
|
12
5
|
|
|
13
6
|
|
|
14
7
|
class Resource(BaseModelObject):
|
|
@@ -21,7 +14,18 @@ T_Resource = TypeVar("T_Resource", bound=Resource)
|
|
|
21
14
|
|
|
22
15
|
|
|
23
16
|
class WriteableResource(Resource, Generic[T_Resource], ABC):
|
|
17
|
+
"""Base class for all writeable data modeling resources."""
|
|
18
|
+
|
|
24
19
|
@abstractmethod
|
|
25
20
|
def as_request(self) -> T_Resource:
|
|
26
21
|
"""Convert the response model to a request model by removing read-only fields."""
|
|
27
22
|
raise NotImplementedError()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class APIResource(Generic[T_Reference], ABC):
|
|
26
|
+
"""Base class for all API data modeling resources."""
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def as_reference(self) -> T_Reference:
|
|
30
|
+
"""Convert the resource to a reference object (identifier)."""
|
|
31
|
+
raise NotImplementedError()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
SPACE_FORMAT_PATTERN = r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$"
|
|
2
2
|
DM_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
|
|
3
3
|
CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,253}[a-zA-Z0-9]?$"
|
|
4
|
-
INSTANCE_ID_PATTERN = r"^[
|
|
4
|
+
INSTANCE_ID_PATTERN = r"^[^\x00]{1,256}$"
|
|
5
5
|
ENUM_VALUE_IDENTIFIER_PATTERN = r"^[_A-Za-z][_0-9A-Za-z]{0,127}$"
|
|
6
6
|
DM_VERSION_PATTERN = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
|
|
7
7
|
FORBIDDEN_ENUM_VALUES = frozenset({"true", "false", "null"})
|
|
@@ -6,8 +6,9 @@ from pydantic import Field, field_validator
|
|
|
6
6
|
from pydantic_core.core_schema import ValidationInfo
|
|
7
7
|
|
|
8
8
|
from cognite.neat._utils.text import humanize_collection
|
|
9
|
+
from cognite.neat._utils.useful_types import BaseModelObject
|
|
9
10
|
|
|
10
|
-
from ._base import
|
|
11
|
+
from ._base import APIResource, Resource, WriteableResource
|
|
11
12
|
from ._constants import (
|
|
12
13
|
CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
|
|
13
14
|
DM_EXTERNAL_ID_PATTERN,
|
|
@@ -36,7 +37,7 @@ class ContainerPropertyDefinition(BaseModelObject):
|
|
|
36
37
|
default=None,
|
|
37
38
|
description="Increment the property based on its highest current value (max value).",
|
|
38
39
|
)
|
|
39
|
-
default_value: str | int | bool | dict[str, Any] | None = Field(
|
|
40
|
+
default_value: str | int | float | bool | dict[str, Any] | None = Field(
|
|
40
41
|
default=None,
|
|
41
42
|
description="Default value to use when you do not specify a value for the property.",
|
|
42
43
|
)
|
|
@@ -53,7 +54,7 @@ class ContainerPropertyDefinition(BaseModelObject):
|
|
|
53
54
|
type: DataType = Field(description="The type of data you can store in this property.")
|
|
54
55
|
|
|
55
56
|
|
|
56
|
-
class Container(Resource, ABC):
|
|
57
|
+
class Container(Resource, APIResource[ContainerReference], ABC):
|
|
57
58
|
space: str = Field(
|
|
58
59
|
description="The workspace for the container, a unique identifier for the space.",
|
|
59
60
|
min_length=1,
|
|
@@ -82,7 +83,6 @@ class Container(Resource, ABC):
|
|
|
82
83
|
)
|
|
83
84
|
properties: dict[str, ContainerPropertyDefinition] = Field(
|
|
84
85
|
description="Set of properties to apply to the container.",
|
|
85
|
-
min_length=1,
|
|
86
86
|
)
|
|
87
87
|
constraints: dict[str, Constraint] | None = Field(
|
|
88
88
|
default=None,
|
|
@@ -136,7 +136,7 @@ class Container(Resource, ABC):
|
|
|
136
136
|
def as_reference(self) -> ContainerReference:
|
|
137
137
|
return ContainerReference(
|
|
138
138
|
space=self.space,
|
|
139
|
-
|
|
139
|
+
external_id=self.external_id,
|
|
140
140
|
)
|
|
141
141
|
|
|
142
142
|
|
|
@@ -2,7 +2,7 @@ from abc import ABC
|
|
|
2
2
|
|
|
3
3
|
from pydantic import Field
|
|
4
4
|
|
|
5
|
-
from ._base import Resource, WriteableResource
|
|
5
|
+
from ._base import APIResource, Resource, WriteableResource
|
|
6
6
|
from ._constants import (
|
|
7
7
|
DM_EXTERNAL_ID_PATTERN,
|
|
8
8
|
DM_VERSION_PATTERN,
|
|
@@ -11,7 +11,7 @@ from ._constants import (
|
|
|
11
11
|
from ._references import DataModelReference, ViewReference
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class DataModel(Resource, ABC):
|
|
14
|
+
class DataModel(Resource, APIResource[DataModelReference], ABC):
|
|
15
15
|
"""Cognite Data Model resource.
|
|
16
16
|
|
|
17
17
|
Data models group and structure views into reusable collections.
|
|
@@ -55,7 +55,7 @@ class DataModel(Resource, ABC):
|
|
|
55
55
|
def as_reference(self) -> DataModelReference:
|
|
56
56
|
return DataModelReference(
|
|
57
57
|
space=self.space,
|
|
58
|
-
|
|
58
|
+
external_id=self.external_id,
|
|
59
59
|
version=self.version,
|
|
60
60
|
)
|
|
61
61
|
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import re
|
|
2
2
|
from abc import ABC
|
|
3
|
+
from collections.abc import Mapping
|
|
3
4
|
from typing import Annotated, Literal
|
|
4
5
|
|
|
5
6
|
from pydantic import Field, TypeAdapter, field_validator
|
|
6
7
|
|
|
7
|
-
from .
|
|
8
|
+
from cognite.neat._utils.auxiliary import get_concrete_subclasses
|
|
9
|
+
from cognite.neat._utils.useful_types import BaseModelObject
|
|
10
|
+
|
|
8
11
|
from ._constants import ENUM_VALUE_IDENTIFIER_PATTERN, FORBIDDEN_ENUM_VALUES, INSTANCE_ID_PATTERN
|
|
9
12
|
from ._references import ContainerReference
|
|
10
13
|
|
|
@@ -181,3 +184,7 @@ DataType = Annotated[
|
|
|
181
184
|
]
|
|
182
185
|
|
|
183
186
|
DataTypeAdapter: TypeAdapter[DataType] = TypeAdapter(DataType)
|
|
187
|
+
|
|
188
|
+
DMS_DATA_TYPES: Mapping[str, type[PropertyTypeDefinition]] = {
|
|
189
|
+
cls_.model_fields["type"].default: cls_ for cls_ in get_concrete_subclasses(PropertyTypeDefinition)
|
|
190
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from typing import TypeAlias, TypeVar
|
|
2
|
+
|
|
3
|
+
from cognite.neat._utils.http_client import ItemBody
|
|
4
|
+
from cognite.neat._utils.useful_types import ReferenceObject
|
|
5
|
+
|
|
6
|
+
from ._container import ContainerRequest
|
|
7
|
+
from ._data_model import DataModelRequest
|
|
8
|
+
from ._space import SpaceRequest
|
|
9
|
+
from ._views import ViewRequest
|
|
10
|
+
|
|
11
|
+
DataModelResource: TypeAlias = SpaceRequest | DataModelRequest | ViewRequest | ContainerRequest
|
|
12
|
+
|
|
13
|
+
T_DataModelResource = TypeVar("T_DataModelResource", bound=DataModelResource)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DataModelBody(ItemBody[ReferenceObject, DataModelResource]):
|
|
17
|
+
def as_ids(self) -> list[ReferenceObject]:
|
|
18
|
+
return [item.as_reference() for item in self.items]
|
|
@@ -2,7 +2,8 @@ from typing import Literal
|
|
|
2
2
|
|
|
3
3
|
from pydantic import Field
|
|
4
4
|
|
|
5
|
-
from .
|
|
5
|
+
from cognite.neat._utils.useful_types import ReferenceObject
|
|
6
|
+
|
|
6
7
|
from ._constants import (
|
|
7
8
|
CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
|
|
8
9
|
DM_EXTERNAL_ID_PATTERN,
|
|
@@ -12,11 +13,17 @@ from ._constants import (
|
|
|
12
13
|
)
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
class ReferenceObject
|
|
16
|
+
class SpaceReference(ReferenceObject):
|
|
17
|
+
space: str = Field(
|
|
18
|
+
description="Id of the space.",
|
|
19
|
+
min_length=1,
|
|
20
|
+
max_length=43,
|
|
21
|
+
pattern=SPACE_FORMAT_PATTERN,
|
|
22
|
+
)
|
|
16
23
|
|
|
17
24
|
|
|
18
25
|
class ContainerReference(ReferenceObject):
|
|
19
|
-
type: Literal["container"] = "container"
|
|
26
|
+
type: Literal["container"] = Field("container", exclude=True)
|
|
20
27
|
space: str = Field(
|
|
21
28
|
description="Id of the space hosting (containing) the container.",
|
|
22
29
|
min_length=1,
|
|
@@ -30,9 +37,12 @@ class ContainerReference(ReferenceObject):
|
|
|
30
37
|
pattern=DM_EXTERNAL_ID_PATTERN,
|
|
31
38
|
)
|
|
32
39
|
|
|
40
|
+
def __str__(self) -> str:
|
|
41
|
+
return f"{self.space}:{self.external_id}"
|
|
42
|
+
|
|
33
43
|
|
|
34
44
|
class ViewReference(ReferenceObject):
|
|
35
|
-
type: Literal["view"] = "view"
|
|
45
|
+
type: Literal["view"] = Field("view", exclude=True)
|
|
36
46
|
space: str = Field(
|
|
37
47
|
description="Id of the space that the view belongs to.",
|
|
38
48
|
min_length=1,
|
|
@@ -51,6 +61,9 @@ class ViewReference(ReferenceObject):
|
|
|
51
61
|
pattern=DM_VERSION_PATTERN,
|
|
52
62
|
)
|
|
53
63
|
|
|
64
|
+
def __str__(self) -> str:
|
|
65
|
+
return f"{self.space}:{self.external_id}(version={self.version})"
|
|
66
|
+
|
|
54
67
|
|
|
55
68
|
class DataModelReference(ReferenceObject):
|
|
56
69
|
space: str = Field(
|
|
@@ -2,13 +2,14 @@ from abc import ABC
|
|
|
2
2
|
|
|
3
3
|
from pydantic import Field, field_validator
|
|
4
4
|
|
|
5
|
+
from cognite.neat._data_model.models.dms._references import SpaceReference
|
|
5
6
|
from cognite.neat.v0.core._utils.text import humanize_collection
|
|
6
7
|
|
|
7
|
-
from ._base import WriteableResource
|
|
8
|
+
from ._base import APIResource, Resource, WriteableResource
|
|
8
9
|
from ._constants import FORBIDDEN_SPACES, SPACE_FORMAT_PATTERN
|
|
9
10
|
|
|
10
11
|
|
|
11
|
-
class Space(
|
|
12
|
+
class Space(Resource, APIResource[SpaceReference], ABC):
|
|
12
13
|
space: str = Field(
|
|
13
14
|
description="The Space identifier (id).",
|
|
14
15
|
min_length=1,
|
|
@@ -25,11 +26,14 @@ class Space(WriteableResource["SpaceRequest"], ABC):
|
|
|
25
26
|
raise ValueError(f"{val!r} is a reserved space. Reserved Spaces: {humanize_collection(FORBIDDEN_SPACES)}")
|
|
26
27
|
return val
|
|
27
28
|
|
|
28
|
-
def
|
|
29
|
-
return
|
|
29
|
+
def as_reference(self) -> SpaceReference:
|
|
30
|
+
return SpaceReference(space=self.space)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class SpaceRequest(Space): ...
|
|
30
34
|
|
|
31
35
|
|
|
32
|
-
class SpaceResponse(Space):
|
|
36
|
+
class SpaceResponse(Space, WriteableResource[Space]):
|
|
33
37
|
created_time: int = Field(
|
|
34
38
|
description="When the space was created. The number of milliseconds since 00:00:00 Thursday, 1 January 1970, "
|
|
35
39
|
"Coordinated Universal Time (UTC), minus leap seconds."
|
|
@@ -40,5 +44,5 @@ class SpaceResponse(Space):
|
|
|
40
44
|
)
|
|
41
45
|
is_global: bool = Field(description="Whether the space is a global space.")
|
|
42
46
|
|
|
43
|
-
|
|
44
|
-
|
|
47
|
+
def as_request(self) -> "SpaceRequest":
|
|
48
|
+
return SpaceRequest.model_validate(self.model_dump(by_alias=True))
|
|
@@ -3,7 +3,9 @@ from typing import Annotated, Literal
|
|
|
3
3
|
|
|
4
4
|
from pydantic import Field, Json, TypeAdapter
|
|
5
5
|
|
|
6
|
-
from .
|
|
6
|
+
from cognite.neat._utils.useful_types import BaseModelObject
|
|
7
|
+
|
|
8
|
+
from ._base import Resource, WriteableResource
|
|
7
9
|
from ._constants import CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN
|
|
8
10
|
from ._data_types import DataType
|
|
9
11
|
from ._references import ContainerDirectReference, ContainerReference, NodeReference, ViewDirectReference, ViewReference
|
|
@@ -49,6 +51,7 @@ class ViewCorePropertyRequest(ViewCoreProperty): ...
|
|
|
49
51
|
|
|
50
52
|
class ConstraintOrIndexState(BaseModelObject):
|
|
51
53
|
nullability: Literal["current", "pending", "failed"] | None = Field(
|
|
54
|
+
None,
|
|
52
55
|
description="""For properties that have isNullable set to false, this field describes the validity of the
|
|
53
56
|
not-null constraint. It is not specified for nullable properties.
|
|
54
57
|
|
|
@@ -58,7 +61,7 @@ Possible values are:
|
|
|
58
61
|
existing nulls was made non-nullable. New null values will still be rejected.
|
|
59
62
|
"current": The constraint is satisfied; all values in the property are not null.
|
|
60
63
|
"pending": The constraint validity has not yet been computed.
|
|
61
|
-
"""
|
|
64
|
+
""",
|
|
62
65
|
)
|
|
63
66
|
|
|
64
67
|
|
|
@@ -147,7 +150,7 @@ class SingleReverseDirectRelationPropertyResponse(
|
|
|
147
150
|
ReverseDirectRelationProperty, WriteableResource[SingleReverseDirectRelationPropertyRequest]
|
|
148
151
|
):
|
|
149
152
|
connection_type: Literal["single_reverse_direct_relation"] = "single_reverse_direct_relation"
|
|
150
|
-
|
|
153
|
+
targets_list: bool = Field(
|
|
151
154
|
description="Whether or not this reverse direct relation targets a list of direct relations.",
|
|
152
155
|
)
|
|
153
156
|
|
|
@@ -159,7 +162,7 @@ class MultiReverseDirectRelationPropertyResponse(
|
|
|
159
162
|
ReverseDirectRelationProperty, WriteableResource[MultiReverseDirectRelationPropertyRequest]
|
|
160
163
|
):
|
|
161
164
|
connection_type: Literal["multi_reverse_direct_relation"] = "multi_reverse_direct_relation"
|
|
162
|
-
|
|
165
|
+
targets_list: bool = Field(
|
|
163
166
|
description="Whether or not this reverse direct relation targets a list of direct relations.",
|
|
164
167
|
)
|
|
165
168
|
|