cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -2,13 +2,23 @@
|
|
|
2
2
|
its sub-models and validators.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
import math
|
|
7
6
|
import sys
|
|
8
7
|
import types
|
|
9
8
|
from abc import ABC, abstractmethod
|
|
10
9
|
from collections.abc import Callable, Hashable, Iterator, MutableSequence, Sequence
|
|
11
|
-
from
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from typing import (
|
|
12
|
+
Annotated,
|
|
13
|
+
Any,
|
|
14
|
+
ClassVar,
|
|
15
|
+
Literal,
|
|
16
|
+
SupportsIndex,
|
|
17
|
+
TypeVar,
|
|
18
|
+
get_args,
|
|
19
|
+
get_origin,
|
|
20
|
+
overload,
|
|
21
|
+
)
|
|
12
22
|
|
|
13
23
|
import pandas as pd
|
|
14
24
|
from pydantic import (
|
|
@@ -23,15 +33,25 @@ from pydantic import (
|
|
|
23
33
|
)
|
|
24
34
|
from pydantic.main import IncEx
|
|
25
35
|
from pydantic_core import core_schema
|
|
26
|
-
|
|
27
|
-
|
|
36
|
+
from rdflib import Namespace, URIRef
|
|
37
|
+
|
|
38
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
39
|
+
from cognite.neat._rules.models._types import (
|
|
40
|
+
ContainerEntityType,
|
|
41
|
+
DataModelExternalIdType,
|
|
42
|
+
DmsPropertyType,
|
|
43
|
+
SpaceType,
|
|
44
|
+
StrListType,
|
|
45
|
+
VersionType,
|
|
46
|
+
ViewEntityType,
|
|
47
|
+
)
|
|
48
|
+
from cognite.neat._rules.models.data_types import DataType
|
|
49
|
+
from cognite.neat._rules.models.entities import EdgeEntity, ReverseConnectionEntity, ViewEntity
|
|
28
50
|
|
|
29
51
|
if sys.version_info >= (3, 11):
|
|
30
52
|
from enum import StrEnum
|
|
31
|
-
from typing import Self
|
|
32
53
|
else:
|
|
33
54
|
from backports.strenum import StrEnum
|
|
34
|
-
from typing_extensions import Self
|
|
35
55
|
|
|
36
56
|
|
|
37
57
|
METADATA_VALUE_MAX_LENGTH = 5120
|
|
@@ -82,18 +102,18 @@ class DataModelType(StrEnum):
|
|
|
82
102
|
enterprise = "enterprise"
|
|
83
103
|
|
|
84
104
|
|
|
105
|
+
class DataModelAspect(StrEnum):
|
|
106
|
+
conceptual = "conceptual"
|
|
107
|
+
logical = "logical"
|
|
108
|
+
physical = "physical"
|
|
109
|
+
|
|
110
|
+
|
|
85
111
|
class RoleTypes(StrEnum):
|
|
86
112
|
domain_expert = "domain expert"
|
|
87
113
|
information = "information architect"
|
|
88
|
-
asset = "asset architect"
|
|
89
114
|
dms = "DMS Architect"
|
|
90
115
|
|
|
91
116
|
|
|
92
|
-
class MatchType(StrEnum):
|
|
93
|
-
exact = "exact"
|
|
94
|
-
partial = "partial"
|
|
95
|
-
|
|
96
|
-
|
|
97
117
|
class SchemaModel(BaseModel):
|
|
98
118
|
model_config: ClassVar[ConfigDict] = ConfigDict(
|
|
99
119
|
populate_by_name=True,
|
|
@@ -125,6 +145,46 @@ class BaseMetadata(SchemaModel):
|
|
|
125
145
|
"""
|
|
126
146
|
|
|
127
147
|
role: ClassVar[RoleTypes]
|
|
148
|
+
aspect: ClassVar[DataModelAspect]
|
|
149
|
+
space: SpaceType = Field(alias="prefix")
|
|
150
|
+
external_id: DataModelExternalIdType = Field(alias="externalId")
|
|
151
|
+
version: VersionType
|
|
152
|
+
|
|
153
|
+
name: str | None = Field(
|
|
154
|
+
None,
|
|
155
|
+
description="Human readable name of the data model",
|
|
156
|
+
min_length=1,
|
|
157
|
+
max_length=255,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
description: str | None = Field(None, min_length=1, max_length=1024)
|
|
161
|
+
|
|
162
|
+
creator: StrListType = Field(
|
|
163
|
+
description=(
|
|
164
|
+
"List of contributors to the data model creation, "
|
|
165
|
+
"typically information architects are considered as contributors."
|
|
166
|
+
),
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
created: datetime = Field(
|
|
170
|
+
description=("Date of the data model creation"),
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
updated: datetime = Field(
|
|
174
|
+
description=("Date of the data model update"),
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
@field_validator("*", mode="before")
|
|
178
|
+
def strip_string(cls, value: Any) -> Any:
|
|
179
|
+
if isinstance(value, str):
|
|
180
|
+
return value.strip()
|
|
181
|
+
return value
|
|
182
|
+
|
|
183
|
+
@field_validator("description", mode="before")
|
|
184
|
+
def nan_as_none(cls, value):
|
|
185
|
+
if isinstance(value, float) and math.isnan(value):
|
|
186
|
+
return None
|
|
187
|
+
return value
|
|
128
188
|
|
|
129
189
|
def to_pandas(self) -> pd.Series:
|
|
130
190
|
"""Converts Metadata to pandas Series."""
|
|
@@ -143,15 +203,30 @@ class BaseMetadata(SchemaModel):
|
|
|
143
203
|
def include_role(self, serializer: Callable) -> dict:
|
|
144
204
|
return {"role": self.role.value, **serializer(self)}
|
|
145
205
|
|
|
146
|
-
@
|
|
206
|
+
@property
|
|
207
|
+
def prefix(self) -> str:
|
|
208
|
+
return self.space
|
|
209
|
+
|
|
147
210
|
def as_identifier(self) -> str:
|
|
148
|
-
"
|
|
149
|
-
raise NotImplementedError()
|
|
211
|
+
return f"{self.prefix}:{self.external_id}"
|
|
150
212
|
|
|
151
|
-
@abstractmethod
|
|
152
213
|
def get_prefix(self) -> str:
|
|
153
|
-
|
|
154
|
-
|
|
214
|
+
return self.prefix
|
|
215
|
+
|
|
216
|
+
@property
|
|
217
|
+
def identifier(self) -> URIRef:
|
|
218
|
+
"""Globally unique identifier for the data model.
|
|
219
|
+
|
|
220
|
+
!!! note
|
|
221
|
+
Unlike namespace, the identifier does not end with "/" or "#".
|
|
222
|
+
|
|
223
|
+
"""
|
|
224
|
+
return DEFAULT_NAMESPACE[f"data-model/verified/{self.aspect}/{self.space}/{self.external_id}/{self.version}"]
|
|
225
|
+
|
|
226
|
+
@property
|
|
227
|
+
def namespace(self) -> Namespace:
|
|
228
|
+
"""Namespace for the data model used for the entities in the data model."""
|
|
229
|
+
return Namespace(f"{self.identifier}/")
|
|
155
230
|
|
|
156
231
|
|
|
157
232
|
class BaseRules(SchemaModel, ABC):
|
|
@@ -169,14 +244,13 @@ class BaseRules(SchemaModel, ABC):
|
|
|
169
244
|
"""
|
|
170
245
|
|
|
171
246
|
metadata: BaseMetadata
|
|
172
|
-
reference: Self | None = Field(None, alias="Reference")
|
|
173
247
|
|
|
174
248
|
@classmethod
|
|
175
249
|
def headers_by_sheet(cls, by_alias: bool = False) -> dict[str, list[str]]:
|
|
176
250
|
"""Returns a list of headers for the model, typically used by ExcelExporter"""
|
|
177
251
|
headers_by_sheet: dict[str, list[str]] = {}
|
|
178
252
|
for field_name, field in cls.model_fields.items():
|
|
179
|
-
if field_name
|
|
253
|
+
if field_name in ["validators_to_skip", "post_validate"]:
|
|
180
254
|
continue
|
|
181
255
|
sheet_name = (field.alias or field_name) if by_alias else field_name
|
|
182
256
|
annotation = field.annotation
|
|
@@ -207,7 +281,6 @@ class BaseRules(SchemaModel, ABC):
|
|
|
207
281
|
def dump(
|
|
208
282
|
self,
|
|
209
283
|
entities_exclude_defaults: bool = True,
|
|
210
|
-
as_reference: bool = False,
|
|
211
284
|
mode: Literal["python", "json"] = "python",
|
|
212
285
|
by_alias: bool = False,
|
|
213
286
|
exclude: IncEx | None = None,
|
|
@@ -224,9 +297,6 @@ class BaseRules(SchemaModel, ABC):
|
|
|
224
297
|
For example, given a class that is dumped as 'my_prefix:MyClass', if the prefix for the rules
|
|
225
298
|
set in metadata.prefix = 'my_prefix', then this class will be dumped as 'MyClass' when this flag is set.
|
|
226
299
|
Defaults to True.
|
|
227
|
-
as_reference (bool, optional): Whether to dump as reference. For Information and DMS rules, this will
|
|
228
|
-
set the reference column/field to the reference of that entity. This is used in the ExcelExporter
|
|
229
|
-
to dump a reference model.
|
|
230
300
|
mode: The mode in which `to_python` should run.
|
|
231
301
|
If mode is 'json', the output will only contain JSON serializable types.
|
|
232
302
|
If mode is 'python', the output may contain non-JSON-serializable Python objects.
|
|
@@ -242,25 +312,11 @@ class BaseRules(SchemaModel, ABC):
|
|
|
242
312
|
if isinstance(value, SheetList):
|
|
243
313
|
value.sort(key=lambda x: x._identifier())
|
|
244
314
|
|
|
245
|
-
context: dict[str, Any] = {
|
|
315
|
+
context: dict[str, Any] = {}
|
|
246
316
|
if entities_exclude_defaults:
|
|
247
317
|
context["metadata"] = self.metadata
|
|
248
318
|
|
|
249
|
-
exclude_input: IncEx | None
|
|
250
|
-
if self.reference is None:
|
|
251
|
-
exclude_input = exclude
|
|
252
|
-
else:
|
|
253
|
-
# If the rules has a reference, we dump that separately with the as_reference flag set to True.
|
|
254
|
-
# We don't want to include the reference in the main dump, so we exclude it here.
|
|
255
|
-
# This is to include whatever is in the exclude set from the user.
|
|
256
|
-
if isinstance(exclude, dict):
|
|
257
|
-
exclude_input = exclude.copy()
|
|
258
|
-
exclude_input["reference"] = {"__all__"} # type: ignore[index]
|
|
259
|
-
elif isinstance(exclude, set):
|
|
260
|
-
exclude_input = exclude.copy()
|
|
261
|
-
exclude_input.add("reference") # type: ignore[arg-type]
|
|
262
|
-
else:
|
|
263
|
-
exclude_input = {"reference"}
|
|
319
|
+
exclude_input: IncEx | None = exclude
|
|
264
320
|
|
|
265
321
|
output = self.model_dump(
|
|
266
322
|
mode=mode,
|
|
@@ -271,20 +327,7 @@ class BaseRules(SchemaModel, ABC):
|
|
|
271
327
|
exclude_defaults=exclude_defaults,
|
|
272
328
|
context=context,
|
|
273
329
|
)
|
|
274
|
-
|
|
275
|
-
if self.reference is not None and not is_reference_user_excluded:
|
|
276
|
-
# If the rules has a reference, we dump that separately with the as_reference flag set to True.
|
|
277
|
-
# Unless the user has explicitly excluded the reference.
|
|
278
|
-
output["Reference" if by_alias else "reference"] = self.reference.dump(
|
|
279
|
-
mode=mode,
|
|
280
|
-
by_alias=by_alias,
|
|
281
|
-
exclude=exclude,
|
|
282
|
-
exclude_none=exclude_none,
|
|
283
|
-
exclude_unset=exclude_unset,
|
|
284
|
-
exclude_defaults=exclude_defaults,
|
|
285
|
-
entities_exclude_defaults=entities_exclude_defaults,
|
|
286
|
-
as_reference=True,
|
|
287
|
-
)
|
|
330
|
+
|
|
288
331
|
return output
|
|
289
332
|
|
|
290
333
|
|
|
@@ -293,6 +336,10 @@ class SheetRow(SchemaModel):
|
|
|
293
336
|
def _identifier(self) -> tuple[Hashable, ...]:
|
|
294
337
|
raise NotImplementedError()
|
|
295
338
|
|
|
339
|
+
def __repr__(self) -> str:
|
|
340
|
+
# Simplified representation of the object for debugging
|
|
341
|
+
return f"{self.__class__.__name__}({self._identifier()})"
|
|
342
|
+
|
|
296
343
|
|
|
297
344
|
T_SheetRow = TypeVar("T_SheetRow", bound=SheetRow)
|
|
298
345
|
|
|
@@ -333,9 +380,9 @@ class SheetList(list, MutableSequence[T_SheetRow]):
|
|
|
333
380
|
def __getitem__(self, index: SupportsIndex) -> T_SheetRow: ...
|
|
334
381
|
|
|
335
382
|
@overload
|
|
336
|
-
def __getitem__(self, index: slice) -> SheetList[T_SheetRow]: ...
|
|
383
|
+
def __getitem__(self, index: slice) -> "SheetList[T_SheetRow]": ...
|
|
337
384
|
|
|
338
|
-
def __getitem__(self, index: SupportsIndex | slice, /) -> T_SheetRow | SheetList[T_SheetRow]:
|
|
385
|
+
def __getitem__(self, index: SupportsIndex | slice, /) -> "T_SheetRow | SheetList[T_SheetRow]":
|
|
339
386
|
if isinstance(index, slice):
|
|
340
387
|
return SheetList[T_SheetRow](super().__getitem__(index))
|
|
341
388
|
return super().__getitem__(index)
|
|
@@ -353,10 +400,19 @@ ExtensionCategoryType = Annotated[
|
|
|
353
400
|
|
|
354
401
|
|
|
355
402
|
# Immutable such that this can be used as a key in a dictionary
|
|
356
|
-
class
|
|
357
|
-
|
|
358
|
-
property_:
|
|
403
|
+
class ContainerProperty(BaseModel, frozen=True):
|
|
404
|
+
container: ContainerEntityType
|
|
405
|
+
property_: DmsPropertyType
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
class ContainerDestinationProperty(ContainerProperty, frozen=True):
|
|
409
|
+
value_type: DataType | ViewEntity
|
|
410
|
+
connection: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None = None
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
class ViewRef(BaseModel, frozen=True):
|
|
414
|
+
view: ViewEntityType
|
|
359
415
|
|
|
360
416
|
|
|
361
|
-
class
|
|
362
|
-
|
|
417
|
+
class ViewProperty(ViewRef, frozen=True):
|
|
418
|
+
property_: DmsPropertyType
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
2
|
+
|
|
1
3
|
from ._rules import DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
|
|
2
4
|
from ._rules_input import (
|
|
3
5
|
DMSInputContainer,
|
|
@@ -8,7 +10,6 @@ from ._rules_input import (
|
|
|
8
10
|
DMSInputRules,
|
|
9
11
|
DMSInputView,
|
|
10
12
|
)
|
|
11
|
-
from ._schema import DMSSchema, PipelineSchema
|
|
12
13
|
|
|
13
14
|
__all__ = [
|
|
14
15
|
"DMSRules",
|
|
@@ -19,7 +20,6 @@ __all__ = [
|
|
|
19
20
|
"DMSContainer",
|
|
20
21
|
"DMSNode",
|
|
21
22
|
"DMSEnum",
|
|
22
|
-
"PipelineSchema",
|
|
23
23
|
"DMSInputRules",
|
|
24
24
|
"DMSInputMetadata",
|
|
25
25
|
"DMSInputView",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections import defaultdict
|
|
3
3
|
from collections.abc import Collection, Hashable, Sequence
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
5
5
|
|
|
6
6
|
from cognite.client.data_classes import data_modeling as dm
|
|
7
7
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex
|
|
@@ -13,15 +13,19 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
13
13
|
ViewPropertyApply,
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
+
from cognite.neat._client.data_classes.data_modeling import (
|
|
17
|
+
ContainerApplyDict,
|
|
18
|
+
NodeApplyDict,
|
|
19
|
+
SpaceApplyDict,
|
|
20
|
+
ViewApplyDict,
|
|
21
|
+
)
|
|
22
|
+
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
16
23
|
from cognite.neat._issues.errors import NeatTypeError, ResourceNotFoundError
|
|
17
24
|
from cognite.neat._issues.warnings import NotSupportedWarning, PropertyNotFoundWarning
|
|
18
25
|
from cognite.neat._issues.warnings.user_modeling import (
|
|
19
26
|
EmptyContainerWarning,
|
|
20
27
|
HasDataFilterOnNoPropertiesViewWarning,
|
|
21
|
-
HasDataFilterOnViewWithReferencesWarning,
|
|
22
|
-
NodeTypeFilterOnParentViewWarning,
|
|
23
28
|
)
|
|
24
|
-
from cognite.neat._rules.models._base_rules import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
25
29
|
from cognite.neat._rules.models.data_types import DataType, Double, Enum, Float
|
|
26
30
|
from cognite.neat._rules.models.entities import (
|
|
27
31
|
ClassEntity,
|
|
@@ -32,15 +36,12 @@ from cognite.neat._rules.models.entities import (
|
|
|
32
36
|
EdgeEntity,
|
|
33
37
|
HasDataFilter,
|
|
34
38
|
NodeTypeFilter,
|
|
35
|
-
ReferenceEntity,
|
|
36
39
|
ReverseConnectionEntity,
|
|
37
40
|
UnitEntity,
|
|
38
41
|
ViewEntity,
|
|
39
42
|
)
|
|
40
|
-
from cognite.neat._utils.cdf.data_classes import ContainerApplyDict, NodeApplyDict, SpaceApplyDict, ViewApplyDict
|
|
41
43
|
|
|
42
44
|
from ._rules import DMSEnum, DMSMetadata, DMSProperty, DMSRules, DMSView
|
|
43
|
-
from ._schema import DMSSchema, PipelineSchema
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
class _DMSExporter:
|
|
@@ -55,78 +56,33 @@ class _DMSExporter:
|
|
|
55
56
|
instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
|
|
56
57
|
"""
|
|
57
58
|
|
|
58
|
-
def __init__(
|
|
59
|
-
self,
|
|
60
|
-
rules: DMSRules,
|
|
61
|
-
include_pipeline: bool = False,
|
|
62
|
-
instance_space: str | None = None,
|
|
63
|
-
):
|
|
64
|
-
self.include_pipeline = include_pipeline
|
|
59
|
+
def __init__(self, rules: DMSRules, instance_space: str | None = None):
|
|
65
60
|
self.instance_space = instance_space
|
|
66
61
|
self.rules = rules
|
|
67
|
-
self._ref_schema =
|
|
62
|
+
self._ref_schema = None
|
|
68
63
|
if self._ref_schema:
|
|
69
64
|
# We skip version as that will always be missing in the reference
|
|
70
65
|
self._ref_views_by_id = {
|
|
71
66
|
dm.ViewId(view.space, view.external_id): view for view in self._ref_schema.views.values()
|
|
72
67
|
}
|
|
73
68
|
else:
|
|
74
|
-
self._ref_views_by_id = {}
|
|
75
|
-
|
|
76
|
-
self.is_addition = (
|
|
77
|
-
rules.metadata.schema_ is SchemaCompleteness.extended
|
|
78
|
-
and rules.metadata.extension is ExtensionCategory.addition
|
|
79
|
-
)
|
|
80
|
-
self.is_reshape = (
|
|
81
|
-
rules.metadata.schema_ is SchemaCompleteness.extended
|
|
82
|
-
and rules.metadata.extension is ExtensionCategory.reshape
|
|
83
|
-
)
|
|
84
|
-
self.is_rebuild = (
|
|
85
|
-
rules.metadata.schema_ is SchemaCompleteness.extended
|
|
86
|
-
and rules.metadata.extension is ExtensionCategory.rebuild
|
|
87
|
-
)
|
|
69
|
+
self._ref_views_by_id = {} # type: ignore
|
|
88
70
|
|
|
89
71
|
def to_schema(self) -> DMSSchema:
|
|
90
72
|
rules = self.rules
|
|
91
73
|
container_properties_by_id, view_properties_by_id = self._gather_properties(list(self.rules.properties))
|
|
92
74
|
|
|
93
|
-
# If we are reshaping or rebuilding, and there are no properties in the current rules, we will
|
|
94
|
-
# include those properties from the last rules.
|
|
95
|
-
if rules.last and (self.is_reshape or self.is_rebuild):
|
|
96
|
-
selected_views = {view.view for view in rules.views}
|
|
97
|
-
selected_properties = [
|
|
98
|
-
prop
|
|
99
|
-
for prop in rules.last.properties
|
|
100
|
-
if prop.view in selected_views and prop.view.as_id() not in view_properties_by_id
|
|
101
|
-
]
|
|
102
|
-
self._update_with_properties(
|
|
103
|
-
selected_properties, container_properties_by_id, view_properties_by_id, include_new_containers=True
|
|
104
|
-
)
|
|
105
|
-
|
|
106
|
-
# We need to include the properties from the last rules as well to create complete containers and view
|
|
107
|
-
# depending on the type of extension.
|
|
108
|
-
if rules.last and self.is_addition:
|
|
109
|
-
selected_properties = [
|
|
110
|
-
prop for prop in rules.last.properties if (prop.view.as_id() in view_properties_by_id)
|
|
111
|
-
]
|
|
112
|
-
self._update_with_properties(selected_properties, container_properties_by_id, view_properties_by_id)
|
|
113
|
-
elif rules.last and (self.is_reshape or self.is_rebuild):
|
|
114
|
-
selected_properties = [
|
|
115
|
-
prop
|
|
116
|
-
for prop in rules.last.properties
|
|
117
|
-
if prop.container and prop.container.as_id() in container_properties_by_id
|
|
118
|
-
]
|
|
119
|
-
self._update_with_properties(selected_properties, container_properties_by_id, None)
|
|
120
|
-
|
|
121
75
|
containers = self._create_containers(container_properties_by_id, rules.enum) # type: ignore[arg-type]
|
|
122
76
|
|
|
123
77
|
view_properties_with_ancestors_by_id = self._gather_properties_with_ancestors(
|
|
124
78
|
view_properties_by_id, rules.views
|
|
125
79
|
)
|
|
126
80
|
|
|
127
|
-
views
|
|
128
|
-
|
|
129
|
-
|
|
81
|
+
views = self._create_views_with_node_types(view_properties_by_id, view_properties_with_ancestors_by_id)
|
|
82
|
+
view_node_type_filters: set[dm.NodeId] = set()
|
|
83
|
+
for dms_view in rules.views:
|
|
84
|
+
if isinstance(dms_view.filter_, NodeTypeFilter):
|
|
85
|
+
view_node_type_filters.update(node.as_id() for node in dms_view.filter_.inner or [])
|
|
130
86
|
if rules.nodes:
|
|
131
87
|
node_types = NodeApplyDict(
|
|
132
88
|
[node.as_node() for node in rules.nodes]
|
|
@@ -136,30 +92,11 @@ class _DMSExporter:
|
|
|
136
92
|
node_types = NodeApplyDict([dm.NodeApply(node.space, node.external_id) for node in view_node_type_filters])
|
|
137
93
|
|
|
138
94
|
last_schema: DMSSchema | None = None
|
|
139
|
-
if self.rules.last:
|
|
140
|
-
last_schema = self.rules.last.as_schema()
|
|
141
|
-
# Remove the views that are in the current model, last + current should equal the full model
|
|
142
|
-
# without any duplicates
|
|
143
|
-
last_schema.views = ViewApplyDict(
|
|
144
|
-
{view_id: view for view_id, view in last_schema.views.items() if view_id not in views}
|
|
145
|
-
)
|
|
146
|
-
last_schema.containers = ContainerApplyDict(
|
|
147
|
-
{
|
|
148
|
-
container_id: container
|
|
149
|
-
for container_id, container in last_schema.containers.items()
|
|
150
|
-
if container_id not in containers
|
|
151
|
-
}
|
|
152
|
-
)
|
|
153
95
|
|
|
154
96
|
views_not_in_model = {view.view.as_id() for view in rules.views if not view.in_model}
|
|
155
|
-
if rules.last and self.is_addition:
|
|
156
|
-
views_not_in_model.update({view.view.as_id() for view in rules.last.views if not view.in_model})
|
|
157
|
-
|
|
158
97
|
data_model = rules.metadata.as_data_model()
|
|
159
98
|
|
|
160
99
|
data_model_views = [view_id for view_id in views if view_id not in views_not_in_model]
|
|
161
|
-
if last_schema and self.is_addition:
|
|
162
|
-
data_model_views.extend([view_id for view_id in last_schema.views if view_id not in views_not_in_model])
|
|
163
100
|
|
|
164
101
|
# Sorting to ensure deterministic order
|
|
165
102
|
data_model.views = sorted(data_model_views, key=lambda v: v.as_tuple()) # type: ignore[union-attr]
|
|
@@ -173,8 +110,6 @@ class _DMSExporter:
|
|
|
173
110
|
containers=containers,
|
|
174
111
|
node_types=node_types,
|
|
175
112
|
)
|
|
176
|
-
if self.include_pipeline:
|
|
177
|
-
return PipelineSchema.from_dms(output, self.instance_space)
|
|
178
113
|
|
|
179
114
|
if self._ref_schema:
|
|
180
115
|
output.reference = self._ref_schema
|
|
@@ -205,19 +140,10 @@ class _DMSExporter:
|
|
|
205
140
|
self,
|
|
206
141
|
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]],
|
|
207
142
|
view_properties_with_ancestors_by_id: dict[dm.ViewId, list[DMSProperty]],
|
|
208
|
-
) ->
|
|
143
|
+
) -> ViewApplyDict:
|
|
209
144
|
input_views = list(self.rules.views)
|
|
210
|
-
if self.rules.last:
|
|
211
|
-
existing = {view.view.as_id() for view in input_views}
|
|
212
|
-
modified_views = [
|
|
213
|
-
v
|
|
214
|
-
for v in self.rules.last.views
|
|
215
|
-
if v.view.as_id() in view_properties_by_id and v.view.as_id() not in existing
|
|
216
|
-
]
|
|
217
|
-
input_views.extend(modified_views)
|
|
218
145
|
|
|
219
146
|
views = ViewApplyDict([dms_view.as_view() for dms_view in input_views])
|
|
220
|
-
dms_view_by_id = {dms_view.view.as_id(): dms_view for dms_view in input_views}
|
|
221
147
|
|
|
222
148
|
for view_id, view in views.items():
|
|
223
149
|
view.properties = {}
|
|
@@ -228,53 +154,12 @@ class _DMSExporter:
|
|
|
228
154
|
if view_property is not None:
|
|
229
155
|
view.properties[prop.view_property] = view_property
|
|
230
156
|
|
|
231
|
-
|
|
232
|
-
unique_node_types: set[dm.NodeId] = set()
|
|
233
|
-
parent_views = {parent for view in views.values() for parent in view.implements or []}
|
|
234
|
-
for view_id, view in views.items():
|
|
235
|
-
dms_view = dms_view_by_id.get(view_id)
|
|
236
|
-
dms_properties = view_properties_by_id.get(view_id, [])
|
|
237
|
-
view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties)
|
|
238
|
-
|
|
239
|
-
view.filter = view_filter.as_dms_filter()
|
|
240
|
-
if isinstance(view_filter, NodeTypeFilter):
|
|
241
|
-
unique_node_types.update(view_filter.nodes)
|
|
242
|
-
if view.as_id() in parent_views:
|
|
243
|
-
warnings.warn(
|
|
244
|
-
NodeTypeFilterOnParentViewWarning(view.as_id()),
|
|
245
|
-
stacklevel=2,
|
|
246
|
-
)
|
|
247
|
-
|
|
248
|
-
elif isinstance(view_filter, HasDataFilter) and data_model_type == DataModelType.solution:
|
|
249
|
-
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
250
|
-
references = {dms_view.reference.as_view_id()}
|
|
251
|
-
elif any(True for prop in dms_properties if isinstance(prop.reference, ReferenceEntity)):
|
|
252
|
-
references = {
|
|
253
|
-
prop.reference.as_view_id()
|
|
254
|
-
for prop in dms_properties
|
|
255
|
-
if isinstance(prop.reference, ReferenceEntity)
|
|
256
|
-
}
|
|
257
|
-
else:
|
|
258
|
-
continue
|
|
259
|
-
warnings.warn(
|
|
260
|
-
HasDataFilterOnViewWithReferencesWarning(view.as_id(), frozenset(references)),
|
|
261
|
-
stacklevel=2,
|
|
262
|
-
)
|
|
263
|
-
|
|
264
|
-
if data_model_type == DataModelType.enterprise:
|
|
265
|
-
# Enterprise Model needs to create node types for all views,
|
|
266
|
-
# as they are expected for the solution model.
|
|
267
|
-
unique_node_types.add(dm.NodeId(space=view.space, external_id=view.external_id))
|
|
268
|
-
|
|
269
|
-
return views, unique_node_types
|
|
157
|
+
return views
|
|
270
158
|
|
|
271
159
|
@classmethod
|
|
272
160
|
def _create_edge_type_from_prop(cls, prop: DMSProperty) -> dm.DirectRelationReference:
|
|
273
161
|
if isinstance(prop.connection, EdgeEntity) and prop.connection.edge_type is not None:
|
|
274
162
|
return prop.connection.edge_type.as_reference()
|
|
275
|
-
elif isinstance(prop.reference, ReferenceEntity):
|
|
276
|
-
ref_view_prop = prop.reference.as_view_property_id()
|
|
277
|
-
return cls._create_edge_type_from_view_id(cast(dm.ViewId, ref_view_prop.source), ref_view_prop.property)
|
|
278
163
|
elif isinstance(prop.value_type, ViewEntity):
|
|
279
164
|
return cls._create_edge_type_from_view_id(prop.view.as_id(), prop.view_property)
|
|
280
165
|
else:
|
|
@@ -298,14 +183,6 @@ class _DMSExporter:
|
|
|
298
183
|
enum_values_by_collection[enum_value.collection].append(enum_value)
|
|
299
184
|
|
|
300
185
|
containers = list(self.rules.containers or [])
|
|
301
|
-
if self.rules.last:
|
|
302
|
-
existing = {container.container.as_id() for container in containers}
|
|
303
|
-
modified_containers = [
|
|
304
|
-
c
|
|
305
|
-
for c in self.rules.last.containers or []
|
|
306
|
-
if c.container.as_id() in container_properties_by_id and c.container.as_id() not in existing
|
|
307
|
-
]
|
|
308
|
-
containers.extend(modified_containers)
|
|
309
186
|
|
|
310
187
|
containers = dm.ContainerApplyList([dms_container.as_container() for dms_container in containers])
|
|
311
188
|
container_to_drop = set()
|
|
@@ -408,17 +285,6 @@ class _DMSExporter:
|
|
|
408
285
|
views: Sequence[DMSView],
|
|
409
286
|
) -> dict[dm.ViewId, list[DMSProperty]]:
|
|
410
287
|
all_view_properties_by_id = view_properties_by_id.copy()
|
|
411
|
-
if self.rules.reference:
|
|
412
|
-
# We need to include t
|
|
413
|
-
ref_view_properties_by_id = self._gather_properties(self.rules.reference.properties)[1]
|
|
414
|
-
for view_id, properties in ref_view_properties_by_id.items():
|
|
415
|
-
if view_id not in all_view_properties_by_id:
|
|
416
|
-
all_view_properties_by_id[view_id] = properties
|
|
417
|
-
else:
|
|
418
|
-
existing_properties = {prop._identifier() for prop in all_view_properties_by_id[view_id]}
|
|
419
|
-
for prop in properties:
|
|
420
|
-
if prop._identifier() not in existing_properties:
|
|
421
|
-
all_view_properties_by_id[view_id].append(prop)
|
|
422
288
|
|
|
423
289
|
view_properties_with_parents_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
|
|
424
290
|
view_by_view_id = {view.view.as_id(): view for view in views}
|
|
@@ -482,37 +348,13 @@ class _DMSExporter:
|
|
|
482
348
|
self,
|
|
483
349
|
view: dm.ViewApply,
|
|
484
350
|
dms_view: DMSView | None,
|
|
485
|
-
|
|
486
|
-
dms_properties: list[DMSProperty],
|
|
487
|
-
) -> DMSFilter:
|
|
351
|
+
) -> DMSFilter | None:
|
|
488
352
|
selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or ""
|
|
489
353
|
|
|
490
354
|
if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty:
|
|
491
355
|
# Has Explicit Filter, use it
|
|
492
356
|
return dms_view.filter_
|
|
493
357
|
|
|
494
|
-
if data_model_type == DataModelType.solution and selected_filter_name in [NodeTypeFilter.name, ""]:
|
|
495
|
-
if (
|
|
496
|
-
dms_view
|
|
497
|
-
and isinstance(dms_view.reference, ReferenceEntity)
|
|
498
|
-
and not dms_properties
|
|
499
|
-
and (ref_view := self._ref_views_by_id.get(dms_view.reference.as_view_id()))
|
|
500
|
-
and ref_view.filter
|
|
501
|
-
):
|
|
502
|
-
# No new properties, only reference, reuse the reference filter
|
|
503
|
-
return DMSFilter.from_dms_filter(ref_view.filter)
|
|
504
|
-
else:
|
|
505
|
-
referenced_node_ids = {
|
|
506
|
-
prop.reference.as_node_entity()
|
|
507
|
-
for prop in dms_properties
|
|
508
|
-
if isinstance(prop.reference, ReferenceEntity)
|
|
509
|
-
}
|
|
510
|
-
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
511
|
-
referenced_node_ids.add(dms_view.reference.as_node_entity())
|
|
512
|
-
|
|
513
|
-
if referenced_node_ids:
|
|
514
|
-
return NodeTypeFilter(inner=list(referenced_node_ids))
|
|
515
|
-
|
|
516
358
|
# Enterprise Model or (Solution + HasData)
|
|
517
359
|
ref_containers = view.referenced_containers()
|
|
518
360
|
if not ref_containers or selected_filter_name == HasDataFilter.name:
|
|
@@ -597,7 +439,7 @@ class _DMSExporter:
|
|
|
597
439
|
(
|
|
598
440
|
prop
|
|
599
441
|
for prop in view_properties_with_ancestors_by_id.get(source_view_id, [])
|
|
600
|
-
if prop.
|
|
442
|
+
if prop.view_property == reverse_prop_id
|
|
601
443
|
),
|
|
602
444
|
None,
|
|
603
445
|
)
|