cognite-neat 0.98.0__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +21 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/analysis/_base.py +8 -4
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/importers/_dms2rules.py +4 -5
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +25 -33
- cognite/neat/_rules/models/__init__.py +1 -1
- cognite/neat/_rules/models/_base_rules.py +22 -12
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +15 -20
- cognite/neat/_rules/models/dms/_rules.py +48 -3
- cognite/neat/_rules/models/dms/_rules_input.py +52 -8
- cognite/neat/_rules/models/dms/_validation.py +10 -5
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +0 -8
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +2 -2
- cognite/neat/_rules/transformers/_converters.py +110 -11
- cognite/neat/_rules/transformers/_mapping.py +105 -30
- cognite/neat/_rules/transformers/_verification.py +5 -2
- cognite/neat/_session/_base.py +49 -8
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +17 -5
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +218 -23
- cognite/neat/_session/_read.py +49 -12
- cognite/neat/_session/_to.py +3 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +8 -3
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +3 -2
- {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +3 -3
- {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +67 -64
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.98.0.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
|
|
|
4
4
|
from collections import Counter, defaultdict
|
|
5
5
|
from collections.abc import Collection, Mapping
|
|
6
6
|
from datetime import date, datetime
|
|
7
|
-
from typing import Literal, TypeVar, cast
|
|
7
|
+
from typing import Literal, TypeVar, cast, overload
|
|
8
8
|
|
|
9
9
|
from cognite.client.data_classes import data_modeling as dms
|
|
10
10
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier, ViewId
|
|
@@ -42,8 +42,10 @@ from cognite.neat._rules.models.entities import (
|
|
|
42
42
|
ContainerEntity,
|
|
43
43
|
DMSUnknownEntity,
|
|
44
44
|
EdgeEntity,
|
|
45
|
+
Entity,
|
|
45
46
|
MultiValueTypeInfo,
|
|
46
47
|
ReverseConnectionEntity,
|
|
48
|
+
T_Entity,
|
|
47
49
|
UnknownEntity,
|
|
48
50
|
ViewEntity,
|
|
49
51
|
)
|
|
@@ -176,6 +178,74 @@ class ToCompliantEntities(RulesTransformer[InformationInputRules, InformationInp
|
|
|
176
178
|
return fixed_definitions
|
|
177
179
|
|
|
178
180
|
|
|
181
|
+
class PrefixEntities(RulesTransformer[InputRules, InputRules]): # type: ignore[misc]
|
|
182
|
+
"""Prefixes all entities with a given prefix."""
|
|
183
|
+
|
|
184
|
+
def __init__(self, prefix: str) -> None:
|
|
185
|
+
self._prefix = prefix
|
|
186
|
+
|
|
187
|
+
def transform(self, rules: InputRules | OutRules[InputRules]) -> ReadRules[InputRules]:
|
|
188
|
+
return ReadRules(self._transform(self._to_rules(rules)), IssueList(), {})
|
|
189
|
+
|
|
190
|
+
def _transform(self, rules: InputRules) -> InputRules:
|
|
191
|
+
rules.metadata.version += f"_prefixed_{self._prefix}"
|
|
192
|
+
|
|
193
|
+
if isinstance(rules, InformationInputRules):
|
|
194
|
+
# Todo Make Not mutate input class
|
|
195
|
+
prefixed_by_class: dict[str, str] = {}
|
|
196
|
+
for cls in rules.classes:
|
|
197
|
+
prefixed = str(self._with_prefix(cls.class_))
|
|
198
|
+
prefixed_by_class[str(cls.class_)] = prefixed
|
|
199
|
+
cls.class_ = prefixed
|
|
200
|
+
for prop in rules.properties:
|
|
201
|
+
prop.class_ = self._with_prefix(prop.class_)
|
|
202
|
+
if str(prop.value_type) in prefixed_by_class:
|
|
203
|
+
prop.value_type = prefixed_by_class[str(prop.value_type)]
|
|
204
|
+
return rules
|
|
205
|
+
elif isinstance(rules, DMSInputRules):
|
|
206
|
+
# Todo not mutate input class new_dms = copy.deepcopy(rules)
|
|
207
|
+
prefixed_by_view: dict[str, str] = {}
|
|
208
|
+
for view in rules.views:
|
|
209
|
+
prefixed = str(self._with_prefix(view.view))
|
|
210
|
+
prefixed_by_view[str(view.view)] = prefixed
|
|
211
|
+
view.view = prefixed
|
|
212
|
+
for dms_prop in rules.properties:
|
|
213
|
+
dms_prop.view = self._with_prefix(dms_prop.view)
|
|
214
|
+
if str(dms_prop.value_type) in prefixed_by_view:
|
|
215
|
+
dms_prop.value_type = prefixed_by_view[str(dms_prop.value_type)]
|
|
216
|
+
if rules.containers:
|
|
217
|
+
for container in rules.containers:
|
|
218
|
+
container.container = self._with_prefix(container.container)
|
|
219
|
+
return rules
|
|
220
|
+
raise NeatValueError(f"Unsupported rules type: {type(rules)}")
|
|
221
|
+
|
|
222
|
+
@overload
|
|
223
|
+
def _with_prefix(self, raw: str) -> str: ...
|
|
224
|
+
|
|
225
|
+
@overload
|
|
226
|
+
def _with_prefix(self, raw: T_Entity) -> T_Entity: ...
|
|
227
|
+
|
|
228
|
+
def _with_prefix(self, raw: str | T_Entity) -> str | T_Entity:
|
|
229
|
+
is_entity_format = not isinstance(raw, str)
|
|
230
|
+
entity = Entity.load(raw)
|
|
231
|
+
output: ClassEntity | ViewEntity | ContainerEntity
|
|
232
|
+
if isinstance(entity, ClassEntity):
|
|
233
|
+
output = ClassEntity(prefix=entity.prefix, suffix=f"{self._prefix}{entity.suffix}", version=entity.version)
|
|
234
|
+
elif isinstance(entity, ViewEntity):
|
|
235
|
+
output = ViewEntity(
|
|
236
|
+
space=entity.space, externalId=f"{self._prefix}{entity.external_id}", version=entity.version
|
|
237
|
+
)
|
|
238
|
+
elif isinstance(entity, ContainerEntity):
|
|
239
|
+
output = ContainerEntity(space=entity.space, externalId=f"{self._prefix}{entity.external_id}")
|
|
240
|
+
elif isinstance(entity, UnknownEntity | Entity):
|
|
241
|
+
return f"{self._prefix}{raw}"
|
|
242
|
+
else:
|
|
243
|
+
raise NeatValueError(f"Unsupported entity type: {type(entity)}")
|
|
244
|
+
if is_entity_format:
|
|
245
|
+
return cast(T_Entity, output)
|
|
246
|
+
return str(output)
|
|
247
|
+
|
|
248
|
+
|
|
179
249
|
class InformationToDMS(ConversionTransformer[InformationRules, DMSRules]):
|
|
180
250
|
"""Converts InformationRules to DMSRules."""
|
|
181
251
|
|
|
@@ -233,10 +303,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
233
303
|
self,
|
|
234
304
|
new_model_id: DataModelIdentifier,
|
|
235
305
|
org_name: str = "My",
|
|
236
|
-
type_: Literal["enterprise", "solution"] = "enterprise",
|
|
306
|
+
type_: Literal["enterprise", "solution", "data_product"] = "enterprise",
|
|
237
307
|
mode: Literal["read", "write"] = "read",
|
|
238
308
|
dummy_property: str = "GUID",
|
|
239
309
|
move_connections: bool = False,
|
|
310
|
+
include: Literal["same-space", "all"] = "same-space",
|
|
240
311
|
):
|
|
241
312
|
self.new_model_id = DataModelId.load(new_model_id)
|
|
242
313
|
if not self.new_model_id.version:
|
|
@@ -247,6 +318,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
247
318
|
self.type_ = type_
|
|
248
319
|
self.dummy_property = dummy_property
|
|
249
320
|
self.move_connections = move_connections
|
|
321
|
+
self.include = include
|
|
250
322
|
|
|
251
323
|
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
252
324
|
# Copy to ensure immutability
|
|
@@ -274,6 +346,16 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
274
346
|
)
|
|
275
347
|
|
|
276
348
|
return self._to_enterprise(reference_model)
|
|
349
|
+
elif self.type_ == "data_product":
|
|
350
|
+
expanded = self._expand_properties(reference_model.model_copy(deep=True))
|
|
351
|
+
if self.include == "same-space":
|
|
352
|
+
expanded.properties = SheetList[DMSProperty](
|
|
353
|
+
[prop for prop in expanded.properties if prop.view.space == expanded.metadata.space]
|
|
354
|
+
)
|
|
355
|
+
expanded.views = SheetList[DMSView](
|
|
356
|
+
[view for view in expanded.views if view.view.space == expanded.metadata.space]
|
|
357
|
+
)
|
|
358
|
+
return self._to_solution(expanded, remove_views_in_other_space=False)
|
|
277
359
|
|
|
278
360
|
else:
|
|
279
361
|
raise NeatValueError(f"Unsupported data model type: {self.type_}")
|
|
@@ -281,7 +363,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
281
363
|
def _has_views_in_multiple_space(self, rules: DMSRules) -> bool:
|
|
282
364
|
return any(view.view.space != rules.metadata.space for view in rules.views)
|
|
283
365
|
|
|
284
|
-
def _to_solution(self, reference_rules: DMSRules) -> JustRules[DMSRules]:
|
|
366
|
+
def _to_solution(self, reference_rules: DMSRules, remove_views_in_other_space: bool = True) -> JustRules[DMSRules]:
|
|
285
367
|
"""For creation of solution data model / rules specifically for mapping over existing containers."""
|
|
286
368
|
|
|
287
369
|
dump = reference_rules.dump()
|
|
@@ -292,16 +374,11 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
292
374
|
dump["metadata"]["external_id"] = self.new_model_id.external_id
|
|
293
375
|
dump["metadata"]["version"] = self.new_model_id.version
|
|
294
376
|
|
|
295
|
-
# dropping reference and last from the dump as they can cause validation
|
|
296
|
-
# issues especially if reference is enterprise model build on top of CDM
|
|
297
|
-
dump.pop("reference", None)
|
|
298
|
-
dump.pop("last", None)
|
|
299
|
-
|
|
300
377
|
# Set implement to NONE for all views
|
|
301
378
|
for view in dump["views"]:
|
|
302
379
|
view["implements"] = None
|
|
303
380
|
|
|
304
|
-
if self._has_views_in_multiple_space(reference_rules):
|
|
381
|
+
if remove_views_in_other_space and self._has_views_in_multiple_space(reference_rules):
|
|
305
382
|
views_to_remove = []
|
|
306
383
|
for view in dump["views"]:
|
|
307
384
|
if ":" in view["view"]:
|
|
@@ -312,7 +389,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
312
389
|
solution_model = DMSRules.model_validate(DMSInputRules.load(dump).dump())
|
|
313
390
|
|
|
314
391
|
# Dropping containers coming from reference model
|
|
315
|
-
solution_model.containers =
|
|
392
|
+
solution_model.containers = None
|
|
316
393
|
|
|
317
394
|
# We want to map properties to existing containers allowing extension
|
|
318
395
|
for prop in solution_model.properties:
|
|
@@ -338,7 +415,7 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
338
415
|
|
|
339
416
|
# Here we add ONLY dummy properties of the solution model and
|
|
340
417
|
# corresponding solution model space containers to hold them
|
|
341
|
-
solution_model.containers
|
|
418
|
+
solution_model.containers = new_containers
|
|
342
419
|
solution_model.properties.extend(new_properties)
|
|
343
420
|
|
|
344
421
|
return JustRules(solution_model)
|
|
@@ -377,6 +454,28 @@ class ToExtension(RulesTransformer[DMSRules, DMSRules]):
|
|
|
377
454
|
|
|
378
455
|
return JustRules(enterprise_model)
|
|
379
456
|
|
|
457
|
+
@staticmethod
|
|
458
|
+
def _expand_properties(rules: DMSRules) -> DMSRules:
|
|
459
|
+
probe = DMSAnalysis(rules)
|
|
460
|
+
ancestor_properties_by_view = probe.classes_with_properties(
|
|
461
|
+
consider_inheritance=True, allow_different_namespace=True
|
|
462
|
+
)
|
|
463
|
+
property_ids_by_view = {
|
|
464
|
+
view: {prop.view_property for prop in properties}
|
|
465
|
+
for view, properties in probe.classes_with_properties(consider_inheritance=False).items()
|
|
466
|
+
}
|
|
467
|
+
for view, property_ids in property_ids_by_view.items():
|
|
468
|
+
ancestor_properties = ancestor_properties_by_view.get(view, [])
|
|
469
|
+
for prop in ancestor_properties:
|
|
470
|
+
if isinstance(prop.connection, ReverseConnectionEntity):
|
|
471
|
+
# If you try to add a reverse direct relation of a parent, it will fail as the ValueType of the
|
|
472
|
+
# original property will point to the parent view, and not the child.
|
|
473
|
+
continue
|
|
474
|
+
if prop.view_property not in property_ids:
|
|
475
|
+
rules.properties.append(prop)
|
|
476
|
+
property_ids.add(prop.view_property)
|
|
477
|
+
return rules
|
|
478
|
+
|
|
380
479
|
def _remove_cognite_affix(self, entity: _T_Entity) -> _T_Entity:
|
|
381
480
|
"""This method removes `Cognite` affix from the entity."""
|
|
382
481
|
new_suffix = entity.suffix.replace("Cognite", self.org_name or "")
|
|
@@ -1,13 +1,16 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from abc import ABC
|
|
2
3
|
from collections import defaultdict
|
|
4
|
+
from functools import cached_property
|
|
5
|
+
from typing import Any, ClassVar, Literal
|
|
3
6
|
|
|
7
|
+
from cognite.neat._issues.errors import NeatValueError
|
|
8
|
+
from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
|
|
4
9
|
from cognite.neat._rules._shared import JustRules, OutRules
|
|
5
|
-
from cognite.neat._rules.models import DMSRules,
|
|
6
|
-
from cognite.neat._rules.models.
|
|
7
|
-
from cognite.neat._rules.models.dms import DMSProperty
|
|
8
|
-
from cognite.neat._rules.models.entities import
|
|
9
|
-
from cognite.neat._rules.models.information import InformationClass
|
|
10
|
-
from cognite.neat._rules.models.mapping import RuleMapping
|
|
10
|
+
from cognite.neat._rules.models import DMSRules, SheetList
|
|
11
|
+
from cognite.neat._rules.models.data_types import Enum
|
|
12
|
+
from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
|
|
13
|
+
from cognite.neat._rules.models.entities import ViewEntity
|
|
11
14
|
|
|
12
15
|
from ._base import RulesTransformer
|
|
13
16
|
|
|
@@ -96,7 +99,7 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
96
99
|
return JustRules(solution)
|
|
97
100
|
|
|
98
101
|
|
|
99
|
-
class RuleMapper(RulesTransformer[
|
|
102
|
+
class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
100
103
|
"""Maps properties and classes using the given mapping.
|
|
101
104
|
|
|
102
105
|
**Note**: This transformer mutates the input rules.
|
|
@@ -106,30 +109,102 @@ class RuleMapper(RulesTransformer[InformationRules, InformationRules]):
|
|
|
106
109
|
|
|
107
110
|
"""
|
|
108
111
|
|
|
109
|
-
|
|
112
|
+
_mapping_fields: ClassVar[frozenset[str]] = frozenset(
|
|
113
|
+
["connection", "value_type", "nullable", "immutable", "is_list", "default", "index", "constraint"]
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def __init__(self, mapping: DMSRules, data_type_conflict: Literal["overwrite"] = "overwrite") -> None:
|
|
110
117
|
self.mapping = mapping
|
|
118
|
+
self.data_type_conflict = data_type_conflict
|
|
119
|
+
|
|
120
|
+
@cached_property
|
|
121
|
+
def _view_by_entity_id(self) -> dict[str, DMSView]:
|
|
122
|
+
return {view.view.external_id: view for view in self.mapping.views}
|
|
111
123
|
|
|
112
|
-
|
|
124
|
+
@cached_property
|
|
125
|
+
def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
|
|
126
|
+
return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
|
|
127
|
+
|
|
128
|
+
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
129
|
+
if self.data_type_conflict != "overwrite":
|
|
130
|
+
raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
|
|
113
131
|
input_rules = self._to_rules(rules)
|
|
132
|
+
new_rules = input_rules.model_copy(deep=True)
|
|
133
|
+
new_rules.metadata.version += "_mapped"
|
|
114
134
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
135
|
+
for view in new_rules.views:
|
|
136
|
+
if mapping_view := self._view_by_entity_id.get(view.view.external_id):
|
|
137
|
+
view.implements = mapping_view.implements
|
|
138
|
+
|
|
139
|
+
for prop in new_rules.properties:
|
|
140
|
+
key = (prop.view.external_id, prop.view_property)
|
|
141
|
+
if key not in self._property_by_view_property:
|
|
142
|
+
continue
|
|
143
|
+
mapping_prop = self._property_by_view_property[key]
|
|
144
|
+
to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
|
|
145
|
+
if conflicts and self.data_type_conflict == "overwrite":
|
|
146
|
+
warnings.warn(
|
|
147
|
+
PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
|
|
148
|
+
stacklevel=2,
|
|
149
|
+
)
|
|
150
|
+
elif conflicts:
|
|
151
|
+
raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
|
|
152
|
+
for field_name, value in to_overwrite.items():
|
|
153
|
+
setattr(prop, field_name, value)
|
|
154
|
+
prop.container = mapping_prop.container
|
|
155
|
+
prop.container_property = mapping_prop.container_property
|
|
156
|
+
|
|
157
|
+
# Add missing views used as value types
|
|
158
|
+
existing_views = {view.view for view in new_rules.views}
|
|
159
|
+
new_value_types = {
|
|
160
|
+
prop.value_type
|
|
161
|
+
for prop in new_rules.properties
|
|
162
|
+
if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
|
|
163
|
+
}
|
|
164
|
+
for new_value_type in new_value_types:
|
|
165
|
+
if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
|
|
166
|
+
new_rules.views.append(mapping_view)
|
|
167
|
+
else:
|
|
168
|
+
warnings.warn(NeatValueWarning(f"View {new_value_type} not found in mapping"), stacklevel=2)
|
|
169
|
+
|
|
170
|
+
# Add missing enums
|
|
171
|
+
existing_enum_collections = {item.collection for item in new_rules.enum or []}
|
|
172
|
+
new_enums = {
|
|
173
|
+
prop.value_type.collection
|
|
174
|
+
for prop in new_rules.properties
|
|
175
|
+
if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
|
|
176
|
+
}
|
|
177
|
+
if new_enums:
|
|
178
|
+
new_rules.enum = new_rules.enum or SheetList[DMSEnum]([])
|
|
179
|
+
for item in self.mapping.enum or []:
|
|
180
|
+
if item.collection in new_enums:
|
|
181
|
+
new_rules.enum.append(item)
|
|
182
|
+
|
|
183
|
+
return JustRules(new_rules)
|
|
184
|
+
|
|
185
|
+
def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
|
|
186
|
+
"""Finds the properties that need to be overwritten and returns them.
|
|
187
|
+
|
|
188
|
+
In addition, conflicting properties are returned. Note that overwriting properties that are
|
|
189
|
+
originally None is not considered a conflict. Thus, you can have properties to overwrite but no
|
|
190
|
+
conflicts.
|
|
191
|
+
|
|
192
|
+
Args:
|
|
193
|
+
prop: The property to compare.
|
|
194
|
+
mapping_prop: The property to compare against.
|
|
195
|
+
|
|
196
|
+
Returns:
|
|
197
|
+
A tuple with the properties to overwrite and the conflicting properties.
|
|
198
|
+
|
|
199
|
+
"""
|
|
200
|
+
to_overwrite: dict[str, Any] = {}
|
|
201
|
+
conflicts: list[str] = []
|
|
202
|
+
for field_name in self._mapping_fields:
|
|
203
|
+
mapping_value = getattr(mapping_prop, field_name)
|
|
204
|
+
source_value = getattr(prop, field_name)
|
|
205
|
+
if mapping_value != source_value:
|
|
206
|
+
to_overwrite[field_name] = mapping_value
|
|
207
|
+
if source_value is not None:
|
|
208
|
+
# These are used for warnings so we use the alias to make it more readable for the user
|
|
209
|
+
conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
|
|
210
|
+
return to_overwrite, conflicts
|
|
@@ -27,8 +27,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
27
27
|
|
|
28
28
|
_rules_cls: type[T_VerifiedRules]
|
|
29
29
|
|
|
30
|
-
def __init__(self, errors: Literal["raise", "continue"]) -> None:
|
|
30
|
+
def __init__(self, errors: Literal["raise", "continue"], post_validate: bool = True) -> None:
|
|
31
31
|
self.errors = errors
|
|
32
|
+
self.post_validate = post_validate
|
|
32
33
|
|
|
33
34
|
def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
|
|
34
35
|
issues = IssueList()
|
|
@@ -39,7 +40,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
39
40
|
verified_rules: T_VerifiedRules | None = None
|
|
40
41
|
with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
|
|
41
42
|
rules_cls = self._get_rules_cls(in_)
|
|
42
|
-
|
|
43
|
+
dumped = in_.dump()
|
|
44
|
+
dumped["post_validate"] = self.post_validate
|
|
45
|
+
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
43
46
|
|
|
44
47
|
if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
|
|
45
48
|
raise issues.as_errors()
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -5,11 +5,13 @@ from cognite.client import CogniteClient
|
|
|
5
5
|
from cognite.client import data_modeling as dm
|
|
6
6
|
|
|
7
7
|
from cognite.neat import _version
|
|
8
|
+
from cognite.neat._client import NeatClient
|
|
8
9
|
from cognite.neat._issues import IssueList, catch_issues
|
|
9
10
|
from cognite.neat._issues.errors import RegexViolationError
|
|
10
11
|
from cognite.neat._rules import importers
|
|
11
12
|
from cognite.neat._rules._shared import ReadRules, VerifiedRules
|
|
12
|
-
from cognite.neat._rules.
|
|
13
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
14
|
+
from cognite.neat._rules.models import DMSInputRules, DMSRules, SheetList
|
|
13
15
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
14
16
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
15
17
|
from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
|
|
@@ -18,10 +20,11 @@ from cognite.neat._store._provenance import (
|
|
|
18
20
|
INSTANCES_ENTITY,
|
|
19
21
|
Change,
|
|
20
22
|
)
|
|
21
|
-
from cognite.neat._utils.auth import _CLIENT_NAME
|
|
22
23
|
|
|
23
24
|
from ._collector import _COLLECTOR, Collector
|
|
25
|
+
from ._drop import DropAPI
|
|
24
26
|
from ._inspect import InspectAPI
|
|
27
|
+
from ._mapping import MappingAPI
|
|
25
28
|
from ._prepare import PrepareAPI
|
|
26
29
|
from ._read import ReadAPI
|
|
27
30
|
from ._set import SetAPI
|
|
@@ -41,19 +44,19 @@ class NeatSession:
|
|
|
41
44
|
verbose: bool = True,
|
|
42
45
|
load_engine: Literal["newest", "cache", "skip"] = "cache",
|
|
43
46
|
) -> None:
|
|
44
|
-
self._client = client
|
|
47
|
+
self._client = NeatClient(client) if client else None
|
|
45
48
|
self._verbose = verbose
|
|
46
49
|
self._state = SessionState(store_type=storage)
|
|
47
|
-
self.read = ReadAPI(self._state,
|
|
48
|
-
self.to = ToAPI(self._state,
|
|
49
|
-
self.prepare = PrepareAPI(self._state, verbose)
|
|
50
|
+
self.read = ReadAPI(self._state, self._client, verbose)
|
|
51
|
+
self.to = ToAPI(self._state, self._client, verbose)
|
|
52
|
+
self.prepare = PrepareAPI(self._client, self._state, verbose)
|
|
50
53
|
self.show = ShowAPI(self._state)
|
|
51
54
|
self.set = SetAPI(self._state, verbose)
|
|
52
55
|
self.inspect = InspectAPI(self._state)
|
|
56
|
+
self.mapping = MappingAPI(self._state)
|
|
57
|
+
self.drop = DropAPI(self._state)
|
|
53
58
|
self.opt = OptAPI()
|
|
54
59
|
self.opt._display()
|
|
55
|
-
if self._client is not None and self._client._config is not None:
|
|
56
|
-
self._client._config.client_name = _CLIENT_NAME
|
|
57
60
|
if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
|
|
58
61
|
print(f"Neat Engine {engine_version} loaded.")
|
|
59
62
|
|
|
@@ -63,6 +66,28 @@ class NeatSession:
|
|
|
63
66
|
|
|
64
67
|
def verify(self) -> IssueList:
|
|
65
68
|
source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
|
|
69
|
+
|
|
70
|
+
reference_rules: DMSInputRules | None = None
|
|
71
|
+
if isinstance(last_unverified_rule.rules, DMSInputRules):
|
|
72
|
+
dms_rules = last_unverified_rule.rules
|
|
73
|
+
views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
|
|
74
|
+
if views_ids or containers_ids:
|
|
75
|
+
if self._client is None:
|
|
76
|
+
raise NeatSessionError(
|
|
77
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
78
|
+
"NEAT needs a client to lookup the definitions. "
|
|
79
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
80
|
+
)
|
|
81
|
+
schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
|
|
82
|
+
|
|
83
|
+
importer = DMSImporter(schema)
|
|
84
|
+
reference_rules = importer.to_rules().rules
|
|
85
|
+
|
|
86
|
+
if reference_rules is not None:
|
|
87
|
+
dms_rules.views.extend(reference_rules.views)
|
|
88
|
+
if dms_rules.containers:
|
|
89
|
+
dms_rules.containers.extend(reference_rules.containers or [])
|
|
90
|
+
|
|
66
91
|
transformer = VerifyAnyRules("continue")
|
|
67
92
|
start = datetime.now(timezone.utc)
|
|
68
93
|
output = transformer.try_transform(last_unverified_rule)
|
|
@@ -78,6 +103,22 @@ class NeatSession:
|
|
|
78
103
|
self._state.data_model.provenance.source_entity(source_id)
|
|
79
104
|
or self._state.data_model.provenance.target_entity(source_id),
|
|
80
105
|
)
|
|
106
|
+
if reference_rules is not None and isinstance(output.rules, DMSRules):
|
|
107
|
+
# Remove the referenced views and containers from the rules
|
|
108
|
+
ref_view_ids = set(reference_rules.as_view_entities())
|
|
109
|
+
if ref_view_ids:
|
|
110
|
+
output.rules.views = SheetList(
|
|
111
|
+
[view for view in output.rules.views if view.view not in ref_view_ids]
|
|
112
|
+
)
|
|
113
|
+
ref_container_ids = reference_rules.as_container_entities()
|
|
114
|
+
if output.rules.containers and ref_container_ids:
|
|
115
|
+
output.rules.containers = SheetList(
|
|
116
|
+
[
|
|
117
|
+
container
|
|
118
|
+
for container in output.rules.containers
|
|
119
|
+
if container.container not in ref_container_ids
|
|
120
|
+
]
|
|
121
|
+
)
|
|
81
122
|
|
|
82
123
|
self._state.data_model.write(output.rules, change)
|
|
83
124
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from rdflib import URIRef
|
|
2
|
+
|
|
3
|
+
from ._state import SessionState
|
|
4
|
+
from .exceptions import session_class_wrapper
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
from rich import print
|
|
8
|
+
except ImportError:
|
|
9
|
+
...
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@session_class_wrapper
|
|
13
|
+
class DropAPI:
|
|
14
|
+
def __init__(self, state: SessionState):
|
|
15
|
+
self._state = state
|
|
16
|
+
|
|
17
|
+
def instances(self, type: str | list[str]) -> None:
|
|
18
|
+
"""Drop instances from the session.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
type: The type of instances to drop.
|
|
22
|
+
"""
|
|
23
|
+
type_list = type if isinstance(type, list) else [type]
|
|
24
|
+
uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types.items())
|
|
25
|
+
selected_uri_by_type: dict[URIRef, str] = {}
|
|
26
|
+
for type_item in type_list:
|
|
27
|
+
if type_item not in uri_type_type:
|
|
28
|
+
print(f"Type {type_item} not found.")
|
|
29
|
+
selected_uri_by_type[uri_type_type[type_item]] = type_item
|
|
30
|
+
|
|
31
|
+
result = self._state.instances.store.queries.drop_types(list(selected_uri_by_type.keys()))
|
|
32
|
+
|
|
33
|
+
for type_uri, count in result.items():
|
|
34
|
+
print(f"Dropped {count} instances of type {selected_uri_by_type[type_uri]}")
|
|
35
|
+
return None
|
|
@@ -11,6 +11,13 @@ from cognite.neat._utils.upload import UploadResult, UploadResultCore, UploadRes
|
|
|
11
11
|
from ._state import SessionState
|
|
12
12
|
from .exceptions import session_class_wrapper
|
|
13
13
|
|
|
14
|
+
try:
|
|
15
|
+
from rich.markdown import Markdown as RichMarkdown
|
|
16
|
+
|
|
17
|
+
RICH_AVAILABLE = True
|
|
18
|
+
except ImportError:
|
|
19
|
+
RICH_AVAILABLE = False
|
|
20
|
+
|
|
14
21
|
|
|
15
22
|
@session_class_wrapper
|
|
16
23
|
class InspectAPI:
|
|
@@ -61,14 +68,19 @@ class InspectIssues:
|
|
|
61
68
|
closest_match = set(difflib.get_close_matches(search, unique_types))
|
|
62
69
|
issues = IssueList([issue for issue in issues if type(issue).__name__ in closest_match])
|
|
63
70
|
|
|
71
|
+
issue_str = "\n".join(
|
|
72
|
+
[f" * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
|
|
73
|
+
)
|
|
74
|
+
markdown_str = f"### {len(issues)} issues found\n\n{issue_str}"
|
|
75
|
+
|
|
64
76
|
if IN_NOTEBOOK:
|
|
65
77
|
from IPython.display import Markdown, display
|
|
66
78
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
79
|
+
display(Markdown(markdown_str))
|
|
80
|
+
elif RICH_AVAILABLE:
|
|
81
|
+
from rich import print
|
|
82
|
+
|
|
83
|
+
print(RichMarkdown(markdown_str))
|
|
72
84
|
|
|
73
85
|
if return_dataframe:
|
|
74
86
|
return issues.to_pandas()
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from datetime import datetime, timezone
|
|
2
|
+
|
|
3
|
+
from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
|
|
4
|
+
from cognite.neat._rules.transformers import RuleMapper
|
|
5
|
+
from cognite.neat._store._provenance import Change
|
|
6
|
+
|
|
7
|
+
from ._state import SessionState
|
|
8
|
+
from .exceptions import session_class_wrapper
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@session_class_wrapper
|
|
12
|
+
class MappingAPI:
|
|
13
|
+
def __init__(self, state: SessionState):
|
|
14
|
+
self._state = state
|
|
15
|
+
|
|
16
|
+
def classic_to_core(self, org_name: str) -> None:
|
|
17
|
+
"""Map classic types to core types.
|
|
18
|
+
|
|
19
|
+
Note this automatically creates an extended CogniteCore model.
|
|
20
|
+
|
|
21
|
+
"""
|
|
22
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
23
|
+
|
|
24
|
+
start = datetime.now(timezone.utc)
|
|
25
|
+
transformer = RuleMapper(load_classic_to_core_mapping(org_name, rules.metadata.space, rules.metadata.version))
|
|
26
|
+
output = transformer.transform(rules)
|
|
27
|
+
end = datetime.now(timezone.utc)
|
|
28
|
+
|
|
29
|
+
change = Change.from_rules_activity(
|
|
30
|
+
output,
|
|
31
|
+
transformer.agent,
|
|
32
|
+
start,
|
|
33
|
+
end,
|
|
34
|
+
"Mapping classic to core",
|
|
35
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
36
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
self._state.data_model.write(output.rules, change)
|