cognite-neat 1.0.16__py3-none-any.whl → 1.0.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_config.py +4 -0
- cognite/neat/_data_model/_analysis.py +159 -1
- cognite/neat/_data_model/_constants.py +8 -1
- cognite/neat/_data_model/_snapshot.py +1 -0
- cognite/neat/_data_model/deployer/deployer.py +21 -7
- cognite/neat/_data_model/models/dms/_view_filter.py +25 -3
- cognite/neat/_data_model/validation/dms/_base.py +1 -0
- cognite/neat/_data_model/validation/dms/_orchestrator.py +4 -0
- cognite/neat/_session/_physical.py +4 -0
- cognite/neat/_session/_result/_deployment/_physical/_changes.py +1 -0
- cognite/neat/_store/_provenance.py +7 -0
- cognite/neat/_store/_store.py +34 -1
- cognite/neat/_version.py +1 -1
- {cognite_neat-1.0.16.dist-info → cognite_neat-1.0.18.dist-info}/METADATA +13 -1
- {cognite_neat-1.0.16.dist-info → cognite_neat-1.0.18.dist-info}/RECORD +16 -16
- {cognite_neat-1.0.16.dist-info → cognite_neat-1.0.18.dist-info}/WHEEL +1 -1
cognite/neat/_config.py
CHANGED
|
@@ -101,6 +101,10 @@ class AlphaFlagConfig(ConfigModel):
|
|
|
101
101
|
default=False,
|
|
102
102
|
description="If enabled, Neat will attempt to automatically fix certain validation issues in the data model.",
|
|
103
103
|
)
|
|
104
|
+
enable_experimental_validators: bool = Field(
|
|
105
|
+
default=False,
|
|
106
|
+
description="If enabled, Neat will run experimental validators that are still in alpha stage.",
|
|
107
|
+
)
|
|
104
108
|
|
|
105
109
|
def __setattr__(self, key: str, value: Any) -> None:
|
|
106
110
|
"""Set attribute value or raise AttributeError."""
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
1
2
|
from itertools import chain
|
|
2
|
-
from typing import Literal, TypeAlias
|
|
3
|
+
from typing import Literal, TypeAlias, TypeVar
|
|
3
4
|
|
|
5
|
+
import networkx as nx
|
|
4
6
|
from pyparsing import cached_property
|
|
5
7
|
|
|
6
8
|
from cognite.neat._data_model._snapshot import SchemaSnapshot
|
|
9
|
+
from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
|
|
7
10
|
from cognite.neat._data_model.models.dms._container import ContainerRequest
|
|
8
11
|
from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
|
|
9
12
|
from cognite.neat._data_model.models.dms._limits import SchemaLimits
|
|
@@ -26,6 +29,7 @@ from cognite.neat._utils.useful_types import ModusOperandi
|
|
|
26
29
|
ViewsByReference: TypeAlias = dict[ViewReference, ViewRequest]
|
|
27
30
|
ContainersByReference: TypeAlias = dict[ContainerReference, ContainerRequest]
|
|
28
31
|
AncestorsByReference: TypeAlias = dict[ViewReference, set[ViewReference]]
|
|
32
|
+
|
|
29
33
|
ReverseToDirectMapping: TypeAlias = dict[
|
|
30
34
|
tuple[ViewReference, str], tuple[ViewReference, ContainerDirectReference | ViewDirectReference]
|
|
31
35
|
]
|
|
@@ -34,6 +38,8 @@ ConnectionEndNodeTypes: TypeAlias = dict[tuple[ViewReference, str], ViewReferenc
|
|
|
34
38
|
|
|
35
39
|
ResourceSource = Literal["auto", "merged", "cdf", "both"]
|
|
36
40
|
|
|
41
|
+
_NodeT = TypeVar("_NodeT", ContainerReference, ViewReference)
|
|
42
|
+
|
|
37
43
|
|
|
38
44
|
class ValidationResources:
|
|
39
45
|
def __init__(
|
|
@@ -378,3 +384,155 @@ class ValidationResources:
|
|
|
378
384
|
connection_end_node_types[(view_ref, prop_ref)] = property_.source
|
|
379
385
|
|
|
380
386
|
return connection_end_node_types
|
|
387
|
+
|
|
388
|
+
@cached_property
|
|
389
|
+
def views_by_container(self) -> dict[ContainerReference, set[ViewReference]]:
|
|
390
|
+
"""Get a mapping from containers to the views that use them.
|
|
391
|
+
|
|
392
|
+
Includes views from both the merged schema and all CDF views to capture
|
|
393
|
+
container-view relationships across the entire CDF environment.
|
|
394
|
+
Uses expanded views to include inherited properties.
|
|
395
|
+
"""
|
|
396
|
+
|
|
397
|
+
# Include all unique views from merged and CDF
|
|
398
|
+
views_by_container: dict[ContainerReference, set[ViewReference]] = defaultdict(set)
|
|
399
|
+
|
|
400
|
+
# Include all unique views from merged and CDF
|
|
401
|
+
all_view_refs = set(self.merged.views.keys()) | set(self.cdf.views.keys())
|
|
402
|
+
|
|
403
|
+
for view_ref in all_view_refs:
|
|
404
|
+
# Use expanded view to include inherited properties
|
|
405
|
+
view = self.expand_view_properties(view_ref)
|
|
406
|
+
if not view:
|
|
407
|
+
continue
|
|
408
|
+
|
|
409
|
+
for container in view.used_containers:
|
|
410
|
+
views_by_container[container].add(view_ref)
|
|
411
|
+
|
|
412
|
+
return dict(views_by_container)
|
|
413
|
+
|
|
414
|
+
@cached_property
|
|
415
|
+
def containers_by_view(self) -> dict[ViewReference, set[ContainerReference]]:
|
|
416
|
+
"""Get a mapping from views to the containers they use.
|
|
417
|
+
|
|
418
|
+
Includes views from both the merged schema and all CDF views.
|
|
419
|
+
Uses expanded views to include inherited properties.
|
|
420
|
+
"""
|
|
421
|
+
containers_by_view: dict[ViewReference, set[ContainerReference]] = {}
|
|
422
|
+
|
|
423
|
+
# Include all unique views from merged and CDF
|
|
424
|
+
all_view_refs = set(self.merged.views.keys()) | set(self.cdf.views.keys())
|
|
425
|
+
|
|
426
|
+
for view_ref in all_view_refs:
|
|
427
|
+
# Use expanded view to include inherited properties
|
|
428
|
+
view = self.expand_view_properties(view_ref)
|
|
429
|
+
if view is not None:
|
|
430
|
+
containers_by_view[view_ref] = view.used_containers
|
|
431
|
+
|
|
432
|
+
return containers_by_view
|
|
433
|
+
|
|
434
|
+
def find_views_mapping_to_containers(self, containers: list[ContainerReference]) -> set[ViewReference]:
|
|
435
|
+
"""Find views that map to all specified containers.
|
|
436
|
+
|
|
437
|
+
That is, the intersection of views that use each of the specified containers.
|
|
438
|
+
|
|
439
|
+
Args:
|
|
440
|
+
containers: List of containers to check
|
|
441
|
+
|
|
442
|
+
Returns:
|
|
443
|
+
Set of views that contain all the specified containers
|
|
444
|
+
|
|
445
|
+
Example:
|
|
446
|
+
Given views V1, V2, V3 and containers C1, C2:
|
|
447
|
+
- V1 uses containers {C1, C2}
|
|
448
|
+
- V2 uses containers {C1}
|
|
449
|
+
- V3 uses containers {C2}
|
|
450
|
+
|
|
451
|
+
find_views_mapping_to_containers([C1, C2]) returns {V1}
|
|
452
|
+
find_views_mapping_to_containers([C1]) returns {V1, V2}
|
|
453
|
+
find_views_mapping_to_containers([C2]) returns {V1, V3}
|
|
454
|
+
"""
|
|
455
|
+
if not containers:
|
|
456
|
+
return set()
|
|
457
|
+
|
|
458
|
+
view_sets = [self.views_by_container.get(c, set()) for c in containers]
|
|
459
|
+
return set.intersection(*view_sets)
|
|
460
|
+
|
|
461
|
+
# =========================================================================
|
|
462
|
+
# Methods used for requires constraint validation
|
|
463
|
+
# =========================================================================
|
|
464
|
+
|
|
465
|
+
@cached_property
|
|
466
|
+
def requires_constraint_graph(self) -> nx.DiGraph:
|
|
467
|
+
"""Build a directed graph of container requires constraints.
|
|
468
|
+
|
|
469
|
+
Nodes are ContainerReferences, edges represent requires constraints.
|
|
470
|
+
An edge A → B means container A requires container B.
|
|
471
|
+
|
|
472
|
+
Includes containers from both merged schema and CDF
|
|
473
|
+
"""
|
|
474
|
+
graph: nx.DiGraph = nx.DiGraph()
|
|
475
|
+
|
|
476
|
+
for container_ref in self.cdf.containers:
|
|
477
|
+
graph.add_node(container_ref)
|
|
478
|
+
for container_ref in self.merged.containers:
|
|
479
|
+
graph.add_node(container_ref)
|
|
480
|
+
|
|
481
|
+
# Add edges for requires constraints from all known containers
|
|
482
|
+
for container_ref in graph.nodes():
|
|
483
|
+
container = self.select_container(container_ref)
|
|
484
|
+
if not container or not container.constraints:
|
|
485
|
+
continue
|
|
486
|
+
for constraint in container.constraints.values():
|
|
487
|
+
if not isinstance(constraint, RequiresConstraintDefinition):
|
|
488
|
+
continue
|
|
489
|
+
graph.add_edge(container_ref, constraint.require)
|
|
490
|
+
|
|
491
|
+
return graph
|
|
492
|
+
|
|
493
|
+
@staticmethod
|
|
494
|
+
def forms_directed_path(nodes: set[_NodeT], graph: nx.DiGraph) -> bool:
|
|
495
|
+
"""Check if nodes form an uninterrupted directed path in the graph.
|
|
496
|
+
|
|
497
|
+
Returns True if there exists a node that can reach all other nodes via
|
|
498
|
+
directed edges in the graph.
|
|
499
|
+
|
|
500
|
+
Args:
|
|
501
|
+
nodes: Set of nodes to check
|
|
502
|
+
graph: Directed graph containing the nodes
|
|
503
|
+
|
|
504
|
+
Returns:
|
|
505
|
+
True if nodes form a directed path (one node reaches all others)
|
|
506
|
+
|
|
507
|
+
Example:
|
|
508
|
+
Given nodes N1, N2, N3 with edges:
|
|
509
|
+
- N1 -> N2
|
|
510
|
+
- N2 -> N3
|
|
511
|
+
|
|
512
|
+
forms_directed_path({N1, N2, N3}) returns True (N1 reaches all others)
|
|
513
|
+
forms_directed_path({N2, N3}) returns True (N2 reaches N3)
|
|
514
|
+
forms_directed_path({N1, N3}) returns False (N1 can't reach N3 without N2)
|
|
515
|
+
"""
|
|
516
|
+
if len(nodes) <= 1:
|
|
517
|
+
return True
|
|
518
|
+
|
|
519
|
+
for candidate in nodes:
|
|
520
|
+
others = nodes - {candidate}
|
|
521
|
+
if others.issubset(nx.descendants(graph, candidate)):
|
|
522
|
+
return True
|
|
523
|
+
|
|
524
|
+
return False
|
|
525
|
+
|
|
526
|
+
@cached_property
|
|
527
|
+
def requires_constraint_cycles(self) -> list[set[ContainerReference]]:
|
|
528
|
+
"""Find all cycles in the requires constraint graph using Tarjan's algorithm.
|
|
529
|
+
|
|
530
|
+
Uses strongly connected components (SCC) to identify cycles efficiently.
|
|
531
|
+
An SCC with more than one node indicates a cycle.
|
|
532
|
+
|
|
533
|
+
Returns:
|
|
534
|
+
List of sets, where each set contains the containers involved in a cycle.
|
|
535
|
+
"""
|
|
536
|
+
sccs = nx.strongly_connected_components(self.requires_constraint_graph)
|
|
537
|
+
# Only SCCs with more than one node represent cycles
|
|
538
|
+
return [scc for scc in sccs if len(scc) > 1]
|
|
@@ -59,7 +59,14 @@ COGNITE_CONCEPTS: tuple[str, ...] = (
|
|
|
59
59
|
*COGNITE_CONCEPTS_3D,
|
|
60
60
|
)
|
|
61
61
|
|
|
62
|
-
COGNITE_SPACES = (
|
|
62
|
+
COGNITE_SPACES = (
|
|
63
|
+
CDF_CDM_SPACE,
|
|
64
|
+
"cdf_360_image_schema",
|
|
65
|
+
"cdf_3d_schema",
|
|
66
|
+
"cdf_apm",
|
|
67
|
+
"cdf_apps_shared",
|
|
68
|
+
"cdf_cdm_3d",
|
|
69
|
+
)
|
|
63
70
|
|
|
64
71
|
# Defaults from https://docs.cognite.com/cdf/dm/dm_reference/dm_limits_and_restrictions#list-size-limits
|
|
65
72
|
|
|
@@ -76,6 +76,7 @@ class SchemaSnapshot(BaseModel, extra="ignore"):
|
|
|
76
76
|
for container_ref, container in merged.containers.items():
|
|
77
77
|
if cdf_container := cdf.containers.get(container_ref):
|
|
78
78
|
container.properties = {**cdf_container.properties, **container.properties}
|
|
79
|
+
container.constraints = {**(cdf_container.constraints or {}), **(container.constraints or {})} or None
|
|
79
80
|
|
|
80
81
|
return merged
|
|
81
82
|
|
|
@@ -4,16 +4,15 @@ from functools import partial
|
|
|
4
4
|
from typing import cast
|
|
5
5
|
|
|
6
6
|
from cognite.neat._client import NeatClient
|
|
7
|
+
from cognite.neat._data_model._constants import COGNITE_SPACES
|
|
7
8
|
from cognite.neat._data_model._shared import OnSuccessResultProducer
|
|
8
9
|
from cognite.neat._data_model._snapshot import SchemaSnapshot
|
|
9
10
|
from cognite.neat._data_model.models.dms import (
|
|
10
|
-
ContainerReference,
|
|
11
11
|
ContainerRequest,
|
|
12
12
|
DataModelBody,
|
|
13
13
|
RequestSchema,
|
|
14
14
|
T_DataModelResource,
|
|
15
15
|
T_ResourceId,
|
|
16
|
-
ViewReference,
|
|
17
16
|
)
|
|
18
17
|
from cognite.neat._utils.collection import chunker_sequence
|
|
19
18
|
from cognite.neat._utils.http_client import (
|
|
@@ -130,7 +129,13 @@ class SchemaDeployer(OnSuccessResultProducer):
|
|
|
130
129
|
def create_deployment_plan(self, snapshot: SchemaSnapshot, data_model: RequestSchema) -> ResourceDeploymentPlanList:
|
|
131
130
|
return ResourceDeploymentPlanList(
|
|
132
131
|
[
|
|
133
|
-
self._create_resource_plan(
|
|
132
|
+
self._create_resource_plan(
|
|
133
|
+
snapshot.spaces,
|
|
134
|
+
data_model.spaces,
|
|
135
|
+
"spaces",
|
|
136
|
+
SpaceDiffer(),
|
|
137
|
+
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
138
|
+
),
|
|
134
139
|
self._create_resource_plan(
|
|
135
140
|
snapshot.containers,
|
|
136
141
|
data_model.containers,
|
|
@@ -150,7 +155,11 @@ class SchemaDeployer(OnSuccessResultProducer):
|
|
|
150
155
|
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
151
156
|
),
|
|
152
157
|
self._create_resource_plan(
|
|
153
|
-
snapshot.data_model,
|
|
158
|
+
snapshot.data_model,
|
|
159
|
+
[data_model.data_model],
|
|
160
|
+
"datamodels",
|
|
161
|
+
DataModelDiffer(),
|
|
162
|
+
skip_criteria=partial(self._skip_resource, model_space=data_model.data_model.space),
|
|
154
163
|
),
|
|
155
164
|
]
|
|
156
165
|
)
|
|
@@ -234,7 +243,7 @@ class SchemaDeployer(OnSuccessResultProducer):
|
|
|
234
243
|
return modified_diffs
|
|
235
244
|
|
|
236
245
|
@classmethod
|
|
237
|
-
def _skip_resource(cls, resource_id:
|
|
246
|
+
def _skip_resource(cls, resource_id: T_ResourceId, model_space: str) -> str | None:
|
|
238
247
|
"""Checks if a resource should be skipped based on its space.
|
|
239
248
|
|
|
240
249
|
Args:
|
|
@@ -244,8 +253,13 @@ class SchemaDeployer(OnSuccessResultProducer):
|
|
|
244
253
|
Returns:
|
|
245
254
|
A reason for skipping if the resource space does not match the model space, otherwise None.
|
|
246
255
|
"""
|
|
247
|
-
if resource_id.space
|
|
248
|
-
return f"Skipping resource
|
|
256
|
+
if resource_id.space in COGNITE_SPACES:
|
|
257
|
+
return f"Skipping resource as it is in the reserved Cognite space '{resource_id.space}'."
|
|
258
|
+
elif resource_id.space != model_space:
|
|
259
|
+
return (
|
|
260
|
+
f"Skipping resource as it is in the space '{resource_id.space}'"
|
|
261
|
+
f" not matching data model space '{model_space}'."
|
|
262
|
+
)
|
|
249
263
|
return None
|
|
250
264
|
|
|
251
265
|
def should_proceed_to_deploy(self, plan: Sequence[ResourceDeploymentPlan]) -> bool:
|
|
@@ -203,7 +203,10 @@ FilterTypes: TypeAlias = Literal[
|
|
|
203
203
|
"not",
|
|
204
204
|
]
|
|
205
205
|
|
|
206
|
+
LegacyFilterTypes: TypeAlias = Literal["invalid"]
|
|
207
|
+
|
|
206
208
|
AVAILABLE_FILTERS: frozenset[str] = frozenset(get_args(FilterTypes))
|
|
209
|
+
LEGACY_FILTERS: frozenset[str] = frozenset(get_args(LegacyFilterTypes))
|
|
207
210
|
|
|
208
211
|
|
|
209
212
|
def _move_filter_key(value: Any) -> Any:
|
|
@@ -246,13 +249,13 @@ def _move_filter_key(value: Any) -> Any:
|
|
|
246
249
|
}
|
|
247
250
|
}
|
|
248
251
|
"""
|
|
252
|
+
# legacy filter which we want to ignore
|
|
253
|
+
if _is_legacy_filter(value):
|
|
254
|
+
return None
|
|
249
255
|
if not isinstance(value, dict):
|
|
250
256
|
return value
|
|
251
257
|
if len(value) != 1:
|
|
252
258
|
raise ValueError("Filter data must have exactly one key.")
|
|
253
|
-
# legacy filter which we want to ignore
|
|
254
|
-
if "invalid" in value:
|
|
255
|
-
return None
|
|
256
259
|
if "filterType" in value:
|
|
257
260
|
# Already in the correct format
|
|
258
261
|
return value
|
|
@@ -283,6 +286,25 @@ def _move_filter_key(value: Any) -> Any:
|
|
|
283
286
|
return value
|
|
284
287
|
|
|
285
288
|
|
|
289
|
+
def _is_legacy_filter(filter_obj: Any) -> bool:
|
|
290
|
+
"""Check if filter is a legacy filter no longer supported by DMS API"""
|
|
291
|
+
|
|
292
|
+
def traverse(obj: Any) -> bool:
|
|
293
|
+
if isinstance(obj, dict):
|
|
294
|
+
for key, value in obj.items():
|
|
295
|
+
if key in LEGACY_FILTERS:
|
|
296
|
+
return True
|
|
297
|
+
if traverse(value):
|
|
298
|
+
return True
|
|
299
|
+
elif isinstance(obj, list):
|
|
300
|
+
for item in obj:
|
|
301
|
+
if traverse(item):
|
|
302
|
+
return True
|
|
303
|
+
return False
|
|
304
|
+
|
|
305
|
+
return traverse(filter_obj)
|
|
306
|
+
|
|
307
|
+
|
|
286
308
|
Filter = Annotated[dict[FilterTypes, FilterData] | None, BeforeValidator(_move_filter_key)]
|
|
287
309
|
|
|
288
310
|
FilterAdapter: TypeAdapter[Filter] = TypeAdapter(Filter)
|
|
@@ -21,6 +21,7 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
|
|
|
21
21
|
limits: SchemaLimits,
|
|
22
22
|
modus_operandi: ModusOperandi = "additive",
|
|
23
23
|
can_run_validator: Callable[[str, type], bool] | None = None,
|
|
24
|
+
enable_alpha_validators: bool = False,
|
|
24
25
|
) -> None:
|
|
25
26
|
super().__init__()
|
|
26
27
|
self._cdf_snapshot = cdf_snapshot
|
|
@@ -28,6 +29,7 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
|
|
|
28
29
|
self._modus_operandi = modus_operandi
|
|
29
30
|
self._can_run_validator = can_run_validator or (lambda code, issue_type: True) # type: ignore
|
|
30
31
|
self._has_run = False
|
|
32
|
+
self._enable_alpha_validators = enable_alpha_validators
|
|
31
33
|
|
|
32
34
|
def run(self, request_schema: RequestSchema) -> None:
|
|
33
35
|
"""Run quality assessment on the DMS data model."""
|
|
@@ -41,6 +43,8 @@ class DmsDataModelValidation(OnSuccessIssuesChecker):
|
|
|
41
43
|
|
|
42
44
|
# Run validators
|
|
43
45
|
for validator in validators:
|
|
46
|
+
if validator.alpha and not self._enable_alpha_validators:
|
|
47
|
+
continue
|
|
44
48
|
if self._can_run_validator(validator.code, validator.issue_type):
|
|
45
49
|
self._issues.extend(validator.run())
|
|
46
50
|
|
|
@@ -108,6 +108,7 @@ class ReadPhysicalDataModel:
|
|
|
108
108
|
cdf_snapshot=self._store.cdf_snapshot,
|
|
109
109
|
limits=self._store.limits,
|
|
110
110
|
can_run_validator=self._config.validation.can_run_validator,
|
|
111
|
+
enable_alpha_validators=self._config.alpha.enable_experimental_validators,
|
|
111
112
|
)
|
|
112
113
|
return self._store.read_physical(reader, on_success)
|
|
113
114
|
|
|
@@ -136,6 +137,7 @@ class ReadPhysicalDataModel:
|
|
|
136
137
|
cdf_snapshot=self._store.cdf_snapshot,
|
|
137
138
|
limits=self._store.limits,
|
|
138
139
|
can_run_validator=self._config.validation.can_run_validator,
|
|
140
|
+
enable_alpha_validators=self._config.alpha.enable_experimental_validators,
|
|
139
141
|
)
|
|
140
142
|
return self._store.read_physical(reader, on_success)
|
|
141
143
|
|
|
@@ -155,6 +157,7 @@ class ReadPhysicalDataModel:
|
|
|
155
157
|
cdf_snapshot=self._store.cdf_snapshot,
|
|
156
158
|
limits=self._store.limits,
|
|
157
159
|
can_run_validator=self._config.validation.can_run_validator,
|
|
160
|
+
enable_alpha_validators=self._config.alpha.enable_experimental_validators,
|
|
158
161
|
)
|
|
159
162
|
|
|
160
163
|
return self._store.read_physical(reader, on_success)
|
|
@@ -177,6 +180,7 @@ class ReadPhysicalDataModel:
|
|
|
177
180
|
cdf_snapshot=self._store.cdf_snapshot,
|
|
178
181
|
limits=self._store.limits,
|
|
179
182
|
can_run_validator=self._config.validation.can_run_validator,
|
|
183
|
+
enable_alpha_validators=self._config.alpha.enable_experimental_validators,
|
|
180
184
|
)
|
|
181
185
|
|
|
182
186
|
return self._store.read_physical(reader, on_success)
|
|
@@ -79,3 +79,10 @@ class Provenance(UserList[Change]):
|
|
|
79
79
|
def provenance_without_failures(self) -> "Provenance":
|
|
80
80
|
"""This method removes all the failed changes from the provenance list."""
|
|
81
81
|
raise NotImplementedError("Not implemented yet")
|
|
82
|
+
|
|
83
|
+
def last_physical_data_model_read(self) -> Change | None:
|
|
84
|
+
"""Get last physical data model read change"""
|
|
85
|
+
for change in reversed(self):
|
|
86
|
+
if change.activity.startswith("to_data_model"):
|
|
87
|
+
return change
|
|
88
|
+
return None
|
cognite/neat/_store/_store.py
CHANGED
|
@@ -11,7 +11,9 @@ from cognite.neat._data_model.deployer.data_classes import DeploymentResult
|
|
|
11
11
|
from cognite.neat._data_model.deployer.deployer import SchemaDeployer
|
|
12
12
|
from cognite.neat._data_model.exporters import DMSExporter, DMSFileExporter
|
|
13
13
|
from cognite.neat._data_model.exporters._api_exporter import DMSAPIExporter
|
|
14
|
+
from cognite.neat._data_model.exporters._table_exporter.exporter import DMSTableExporter
|
|
14
15
|
from cognite.neat._data_model.importers import DMSImporter, DMSTableImporter
|
|
16
|
+
from cognite.neat._data_model.importers._api_importer import DMSAPIImporter
|
|
15
17
|
from cognite.neat._data_model.models.dms import RequestSchema as PhysicalDataModel
|
|
16
18
|
from cognite.neat._data_model.models.dms._limits import SchemaLimits
|
|
17
19
|
from cognite.neat._exceptions import DataModelImportException
|
|
@@ -62,7 +64,9 @@ class NeatStore:
|
|
|
62
64
|
else:
|
|
63
65
|
activity = writer.export
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
data_model = self._gather_data_model(writer)
|
|
68
|
+
|
|
69
|
+
change, _ = self._do_activity(activity, on_success, data_model=data_model, **kwargs)
|
|
66
70
|
|
|
67
71
|
if not change.issues:
|
|
68
72
|
change.target_entity = "ExternalEntity"
|
|
@@ -79,6 +83,35 @@ class NeatStore:
|
|
|
79
83
|
# Update CDF snapshot after successful deployment
|
|
80
84
|
self.cdf_snapshot = SchemaSnapshot.fetch_entire_cdf(self._client)
|
|
81
85
|
|
|
86
|
+
def _gather_data_model(self, writer: DMSExporter) -> PhysicalDataModel:
|
|
87
|
+
"""Gather the current physical data model from the store
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
writer (DMSExporter): The exporter that will be used to write the data model.
|
|
91
|
+
"""
|
|
92
|
+
# getting provenance of the last successful physical data model read
|
|
93
|
+
change = self.provenance.last_physical_data_model_read()
|
|
94
|
+
|
|
95
|
+
if not change:
|
|
96
|
+
raise RuntimeError("No successful physical data model read found in provenance.")
|
|
97
|
+
|
|
98
|
+
# We do not want to modify the data model for API representations
|
|
99
|
+
if not (change.agent == DMSAPIImporter.__name__ and isinstance(writer, DMSTableExporter)):
|
|
100
|
+
return self.physical_data_model[-1]
|
|
101
|
+
|
|
102
|
+
# This will handle data model that are partially and require to be converted to
|
|
103
|
+
# tabular representation to include all containers referenced by views.
|
|
104
|
+
copy = self.physical_data_model[-1].model_copy(deep=True)
|
|
105
|
+
container_refs = {container.as_reference() for container in copy.containers}
|
|
106
|
+
|
|
107
|
+
for view in copy.views:
|
|
108
|
+
for container in view.used_containers:
|
|
109
|
+
if container not in container_refs and (cdf_container := self.cdf_snapshot.containers.get(container)):
|
|
110
|
+
copy.containers.append(cdf_container)
|
|
111
|
+
container_refs.add(container)
|
|
112
|
+
|
|
113
|
+
return copy
|
|
114
|
+
|
|
82
115
|
def _can_agent_do_activity(self, agent: Agents) -> None:
|
|
83
116
|
"""Validate if activity can be performed in the current state and considering provenance"""
|
|
84
117
|
if not self.state.can_transition(agent):
|
cognite/neat/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "1.0.
|
|
1
|
+
__version__ = "1.0.18"
|
|
2
2
|
__engine__ = "^2.0.4"
|
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cognite-neat
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.18
|
|
4
4
|
Summary: Knowledge graph transformation
|
|
5
5
|
Author: Nikola Vasiljevic, Anders Albert
|
|
6
6
|
Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>
|
|
7
7
|
License-Expression: Apache-2.0
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Information Technology
|
|
11
|
+
Classifier: Intended Audience :: Customer Service
|
|
12
|
+
Classifier: Topic :: Database
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
8
20
|
Requires-Dist: pandas>=1.5.3,<3.0.0
|
|
9
21
|
Requires-Dist: cognite-sdk>=7.83.0,<8.0.0
|
|
10
22
|
Requires-Dist: rdflib>=7.0.0,<8.0.0
|
|
@@ -9,13 +9,13 @@ cognite/neat/_client/data_model_api.py,sha256=ogVHOabQ3HTqWaaoiGClmbtYdP-pl6DPN2
|
|
|
9
9
|
cognite/neat/_client/spaces_api.py,sha256=xHtSMt_2k2YwZ5_8kH2dfa7fWxQQrky7wra4Ar2jwqs,4111
|
|
10
10
|
cognite/neat/_client/statistics_api.py,sha256=HcYb2nNC9M_iaI1xyjjLn2Cz1tcyu7BJeaqVps79tg4,773
|
|
11
11
|
cognite/neat/_client/views_api.py,sha256=Qzk_wiLtaWszxCQFDBoWCH1yDc4GOEJsVOcL061rcK0,5639
|
|
12
|
-
cognite/neat/_config.py,sha256=
|
|
12
|
+
cognite/neat/_config.py,sha256=ZvCkcaRVAvH4-ClvinoWaLWhRJpRByqdvncGFsf5gLk,9886
|
|
13
13
|
cognite/neat/_data_model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
cognite/neat/_data_model/_analysis.py,sha256=
|
|
15
|
-
cognite/neat/_data_model/_constants.py,sha256=
|
|
14
|
+
cognite/neat/_data_model/_analysis.py,sha256=lPiXF0td4y5TudrR1KPH5OUZ_YNBz8tO6cLg3RnJwEg,22413
|
|
15
|
+
cognite/neat/_data_model/_constants.py,sha256=E2axzdYjsIy7lTHsjW91wsv6r-pUwko8g6K8C_oRnxk,1707
|
|
16
16
|
cognite/neat/_data_model/_identifiers.py,sha256=2l_bCtuE6TVZLCnzV7hhAUTP0kU6ji4QlIK-JhRK1fM,1922
|
|
17
17
|
cognite/neat/_data_model/_shared.py,sha256=H0gFqa8tKFNWuvdat5jL6OwySjCw3aQkLPY3wtb9Wrw,1302
|
|
18
|
-
cognite/neat/_data_model/_snapshot.py,sha256=
|
|
18
|
+
cognite/neat/_data_model/_snapshot.py,sha256=JBaKmL0Tmprz59SZ1JeB49BPMB8Hqa-OAOt0Bai8cw4,6305
|
|
19
19
|
cognite/neat/_data_model/deployer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
cognite/neat/_data_model/deployer/_differ.py,sha256=1ircRBCoaFooSzMTmTZBTORHeAhDa8YtDEnVwBo6TUI,4742
|
|
21
21
|
cognite/neat/_data_model/deployer/_differ_container.py,sha256=mcy7PhUOfnvAxnZWNoeNRmiXa8ovIn0W6YoqfzVYyiQ,14665
|
|
@@ -23,7 +23,7 @@ cognite/neat/_data_model/deployer/_differ_data_model.py,sha256=iA7Xp-7NRvzZJXLLp
|
|
|
23
23
|
cognite/neat/_data_model/deployer/_differ_space.py,sha256=J_AaqiseLpwQsOkKc7gmho4U2oSWAGVeEdQNepZiWw0,343
|
|
24
24
|
cognite/neat/_data_model/deployer/_differ_view.py,sha256=g1xHwsoxFUaTOTtQa19nntKF3rxFzc2FxpKKFAUN_NE,11412
|
|
25
25
|
cognite/neat/_data_model/deployer/data_classes.py,sha256=cq86u7wuKCcvH-A_cGI_gWzlQCTIG6mrXG2MahdiGco,27299
|
|
26
|
-
cognite/neat/_data_model/deployer/deployer.py,sha256=
|
|
26
|
+
cognite/neat/_data_model/deployer/deployer.py,sha256=lEdTh4jOwTxLkSEx-SlcnXUZyPZCUtIzop1zhAe2s44,19681
|
|
27
27
|
cognite/neat/_data_model/exporters/__init__.py,sha256=AskjmB_0Vqib4kN84bWt8-M8nO42QypFf-l-E8oA5W8,482
|
|
28
28
|
cognite/neat/_data_model/exporters/_api_exporter.py,sha256=G9cqezy_SH8VdhW4o862qBHh_QcbzfP6O1Yyjvdpeog,1579
|
|
29
29
|
cognite/neat/_data_model/exporters/_base.py,sha256=rG_qAU5i5Hh5hUMep2UmDFFZID4x3LEenL6Z5C6N8GQ,646
|
|
@@ -60,7 +60,7 @@ cognite/neat/_data_model/models/dms/_references.py,sha256=Mx_nxfvOrvAx7nvebhhbFw
|
|
|
60
60
|
cognite/neat/_data_model/models/dms/_schema.py,sha256=2JFLcm52smzPdtZ69Lf02UbYAD8I_hpRbI7ZAzdxJJs,641
|
|
61
61
|
cognite/neat/_data_model/models/dms/_space.py,sha256=3KvWg0bVuLpgQwhkDbsJ53ZMMmK0cKUgfyDRrSrERko,1904
|
|
62
62
|
cognite/neat/_data_model/models/dms/_types.py,sha256=5-cgC53AG186OZUqkltv7pMjcGNLuH7Etbn8IUcgk1c,447
|
|
63
|
-
cognite/neat/_data_model/models/dms/_view_filter.py,sha256=
|
|
63
|
+
cognite/neat/_data_model/models/dms/_view_filter.py,sha256=XxMffUH5kYtcg0xHgyUsY4nueWRoJu2CoJtOU7wbH4Y,11274
|
|
64
64
|
cognite/neat/_data_model/models/dms/_view_property.py,sha256=nJBPmw4KzJOdaQmvRfCE3A4FL-E13OsNUEufI64vLKo,9271
|
|
65
65
|
cognite/neat/_data_model/models/dms/_views.py,sha256=oWSqp9_tBNwKQgj4xKICQKE3f89Ya6Xyjcms-I66zN0,8931
|
|
66
66
|
cognite/neat/_data_model/models/entities/__init__.py,sha256=7dDyES7fYl9LEREal59F038RdEvfGRpUOc6n_MtSgjU,836
|
|
@@ -72,12 +72,12 @@ cognite/neat/_data_model/models/entities/_parser.py,sha256=zef_pSDZYMZrJl4IKreFD
|
|
|
72
72
|
cognite/neat/_data_model/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
73
|
cognite/neat/_data_model/validation/dms/__init__.py,sha256=kKD18-Bg_G-w11Cs7Wv_TKV0C_q62Pm2RKLpOz27ar4,2642
|
|
74
74
|
cognite/neat/_data_model/validation/dms/_ai_readiness.py,sha256=bffMQJ5pqumU5P3KaEdQP67OO5eMKqzN2BAWbUjG6KE,16143
|
|
75
|
-
cognite/neat/_data_model/validation/dms/_base.py,sha256=
|
|
75
|
+
cognite/neat/_data_model/validation/dms/_base.py,sha256=G_gMPTgKwyBW62UcCkKIBVHWp9ufAZPJ2p7o69_dJI0,820
|
|
76
76
|
cognite/neat/_data_model/validation/dms/_connections.py,sha256=-kUXf2_3V50ckxwXRwJoTHsKkS5zxiBKkkkHg8Dm4WI,30353
|
|
77
77
|
cognite/neat/_data_model/validation/dms/_consistency.py,sha256=IKSUoRQfQQcsymviESW9VuTFX7jsZMXfsObeZHPdov4,2435
|
|
78
78
|
cognite/neat/_data_model/validation/dms/_containers.py,sha256=5Lka1Cg-SaP9Ued0cku0leG1Sjx76JQ9XcBZK_dtfuM,8520
|
|
79
79
|
cognite/neat/_data_model/validation/dms/_limits.py,sha256=U7z8sN-kAyJsF5hYHPNBBg25Fvz1F8njhzYVSQOIiOU,14779
|
|
80
|
-
cognite/neat/_data_model/validation/dms/_orchestrator.py,sha256=
|
|
80
|
+
cognite/neat/_data_model/validation/dms/_orchestrator.py,sha256=qiuUSUmNhekFyBARUUO2yhG-X9AeU_LL49UrJ65JXFA,2964
|
|
81
81
|
cognite/neat/_data_model/validation/dms/_views.py,sha256=Q0x7jdG69-AVc93VrwdZ1_rFHpq-I-OG98puM4lcweE,5068
|
|
82
82
|
cognite/neat/_exceptions.py,sha256=ox-5hXpee4UJlPE7HpuEHV2C96aLbLKo-BhPDoOAzhA,1650
|
|
83
83
|
cognite/neat/_issues.py,sha256=wH1mnkrpBsHUkQMGUHFLUIQWQlfJ_qMfdF7q0d9wNhY,1871
|
|
@@ -94,11 +94,11 @@ cognite/neat/_session/_html/templates/__init__.py,sha256=hgufJuBxUZ2nLCMTCxGixmk
|
|
|
94
94
|
cognite/neat/_session/_html/templates/deployment.html,sha256=aLDXMbF3pcSqnCpUYVGmIWfqU2jyYUUTaGfpSHRLzdU,3715
|
|
95
95
|
cognite/neat/_session/_html/templates/issues.html,sha256=zjhkJcPK0hMp_ZKJ9RCf88tuZxQyTYRPxzpqx33Nkt0,1661
|
|
96
96
|
cognite/neat/_session/_issues.py,sha256=E8UQeSJURg2dm4MF1pfD9dp-heSRT7pgQZgKlD1-FGs,2723
|
|
97
|
-
cognite/neat/_session/_physical.py,sha256=
|
|
97
|
+
cognite/neat/_session/_physical.py,sha256=uQ1ny5Sy3mvkSIdT5kaCwlDCA1W2U-RQouITNAu7TE0,12433
|
|
98
98
|
cognite/neat/_session/_result/__init__.py,sha256=8A0BKgsqnjxkiHUlCpHBNl3mrFWtyjaWYnh0jssE6QU,50
|
|
99
99
|
cognite/neat/_session/_result/_deployment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
100
100
|
cognite/neat/_session/_result/_deployment/_physical/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
101
|
-
cognite/neat/_session/_result/_deployment/_physical/_changes.py,sha256=
|
|
101
|
+
cognite/neat/_session/_result/_deployment/_physical/_changes.py,sha256=CmFWxAMtVj-iYlLuMsP94UfMzOqXma1Tv0mzQwCd2jQ,6294
|
|
102
102
|
cognite/neat/_session/_result/_deployment/_physical/_statistics.py,sha256=aOXZTSOGmVggERB4mKaqQEUx40vrxiN9ZkwjKU1555A,6324
|
|
103
103
|
cognite/neat/_session/_result/_deployment/_physical/serializer.py,sha256=BbCUb7_C75enMimGgTPg-ZW1QGcRhPDf5dNwpWdM790,1272
|
|
104
104
|
cognite/neat/_session/_result/_result.py,sha256=OnjIJQSnIzYr-IE50rq7z3CVO0LpVE8tAaZN7XPjlKw,1188
|
|
@@ -112,8 +112,8 @@ cognite/neat/_state_machine/__init__.py,sha256=wrtQUHETiLzYM0pFo7JC6pJCiXetHADQb
|
|
|
112
112
|
cognite/neat/_state_machine/_base.py,sha256=-ZpeAhM6l6N6W70dET25tAzOxaaK5aa474eabwZVzjA,1112
|
|
113
113
|
cognite/neat/_state_machine/_states.py,sha256=nmj4SmunpDYcBsNx8A284xnXGS43wuUuWpMMORha2DE,1170
|
|
114
114
|
cognite/neat/_store/__init__.py,sha256=TvM9CcFbtOSrxydPAuJi6Bv_iiGard1Mxfx42ZFoTl0,55
|
|
115
|
-
cognite/neat/_store/_provenance.py,sha256=
|
|
116
|
-
cognite/neat/_store/_store.py,sha256=
|
|
115
|
+
cognite/neat/_store/_provenance.py,sha256=1zzRDWjR9twZu2jVyIG3UdYdIXtQKJ7uF8a0hV7LEuA,3368
|
|
116
|
+
cognite/neat/_store/_store.py,sha256=Jibx6kKAH8V6iUOegDOcSLTplGLxmuXCLfjbdff_wvI,9367
|
|
117
117
|
cognite/neat/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
118
118
|
cognite/neat/_utils/_reader.py,sha256=9dXrODNNqWU0Gx1zXjRTOiiByFuDZlpQkQEzx3HAxYQ,5390
|
|
119
119
|
cognite/neat/_utils/auxiliary.py,sha256=Cx-LP8dfN782R3iUcm--q26zdzQ0k_RFnVbJ0bwVZMI,1345
|
|
@@ -316,9 +316,9 @@ cognite/neat/_v0/session/_template.py,sha256=BNcvrW5y7LWzRM1XFxZkfR1Nc7e8UgjBClH
|
|
|
316
316
|
cognite/neat/_v0/session/_to.py,sha256=AnsRSDDdfFyYwSgi0Z-904X7WdLtPfLlR0x1xsu_jAo,19447
|
|
317
317
|
cognite/neat/_v0/session/_wizard.py,sha256=baPJgXAAF3d1bn4nbIzon1gWfJOeS5T43UXRDJEnD3c,1490
|
|
318
318
|
cognite/neat/_v0/session/exceptions.py,sha256=jv52D-SjxGfgqaHR8vnpzo0SOJETIuwbyffSWAxSDJw,3495
|
|
319
|
-
cognite/neat/_version.py,sha256=
|
|
319
|
+
cognite/neat/_version.py,sha256=Buh8SWFXU0sxa-m7s9PiJfRvoR-zxpBJB4QrFLZmqHk,45
|
|
320
320
|
cognite/neat/legacy.py,sha256=eI2ecxOV8ilGHyLZlN54ve_abtoK34oXognkFv3yvF0,219
|
|
321
321
|
cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
322
|
-
cognite_neat-1.0.
|
|
323
|
-
cognite_neat-1.0.
|
|
324
|
-
cognite_neat-1.0.
|
|
322
|
+
cognite_neat-1.0.18.dist-info/WHEEL,sha256=eycQt0QpYmJMLKpE3X9iDk8R04v2ZF0x82ogq-zP6bQ,79
|
|
323
|
+
cognite_neat-1.0.18.dist-info/METADATA,sha256=7Lg6EUBwa0BYxDgRz8iNpZY8kim0xsRIqRCDSHhYLFo,6689
|
|
324
|
+
cognite_neat-1.0.18.dist-info/RECORD,,
|