cognite-neat 1.0.33__py3-none-any.whl → 1.0.35__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_data_model/_analysis.py +94 -2
- cognite/neat/_data_model/models/conceptual/_concept.py +1 -1
- cognite/neat/_data_model/rules/dms/__init__.py +2 -0
- cognite/neat/_data_model/rules/dms/_connections.py +55 -196
- cognite/neat/_data_model/rules/dms/_containers.py +1 -1
- cognite/neat/_data_model/rules/dms/_performance.py +75 -0
- cognite/neat/_version.py +1 -1
- {cognite_neat-1.0.33.dist-info → cognite_neat-1.0.35.dist-info}/METADATA +1 -1
- {cognite_neat-1.0.33.dist-info → cognite_neat-1.0.35.dist-info}/RECORD +10 -10
- {cognite_neat-1.0.33.dist-info → cognite_neat-1.0.35.dist-info}/WHEEL +1 -1
|
@@ -2,17 +2,17 @@ import math
|
|
|
2
2
|
from collections import defaultdict
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from enum import Enum
|
|
5
|
+
from functools import cached_property
|
|
5
6
|
from itertools import chain, combinations
|
|
6
7
|
from typing import Literal, TypeAlias, TypeVar
|
|
7
8
|
|
|
8
9
|
import networkx as nx
|
|
9
|
-
from pyparsing import cached_property
|
|
10
10
|
|
|
11
11
|
from cognite.neat._client.data_classes import SpaceStatisticsResponse
|
|
12
12
|
from cognite.neat._data_model._constants import COGNITE_SPACES
|
|
13
13
|
from cognite.neat._data_model._snapshot import SchemaSnapshot
|
|
14
14
|
from cognite.neat._data_model.models.dms._constraints import RequiresConstraintDefinition
|
|
15
|
-
from cognite.neat._data_model.models.dms._container import ContainerRequest
|
|
15
|
+
from cognite.neat._data_model.models.dms._container import ContainerPropertyDefinition, ContainerRequest
|
|
16
16
|
from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
|
|
17
17
|
from cognite.neat._data_model.models.dms._limits import SchemaLimits
|
|
18
18
|
from cognite.neat._data_model.models.dms._references import (
|
|
@@ -55,6 +55,21 @@ class RequiresChangeStatus(Enum):
|
|
|
55
55
|
NO_MODIFIABLE_CONTAINERS = "no_modifiable_containers" # All containers are immutable
|
|
56
56
|
|
|
57
57
|
|
|
58
|
+
@dataclass
|
|
59
|
+
class ResolvedReverseDirectRelation:
|
|
60
|
+
"""Resolved context for a reverse direct relation, including container-level information."""
|
|
61
|
+
|
|
62
|
+
reverse_view_ref: ViewReference
|
|
63
|
+
reverse_property_id: str
|
|
64
|
+
direct_view_ref: ViewReference
|
|
65
|
+
through_property_id: str
|
|
66
|
+
direct_property: ViewCorePropertyRequest
|
|
67
|
+
container_ref: ContainerReference
|
|
68
|
+
container_property_id: str
|
|
69
|
+
container: ContainerRequest | None
|
|
70
|
+
container_property: ContainerPropertyDefinition | None
|
|
71
|
+
|
|
72
|
+
|
|
58
73
|
@dataclass
|
|
59
74
|
class RequiresChangesForView:
|
|
60
75
|
"""Result of computing requires constraint changes for a view."""
|
|
@@ -365,6 +380,83 @@ class ValidationResources:
|
|
|
365
380
|
|
|
366
381
|
return bidirectional_connections
|
|
367
382
|
|
|
383
|
+
@staticmethod
|
|
384
|
+
def normalize_through_reference(
|
|
385
|
+
source_view_ref: ViewReference, through: ContainerDirectReference | ViewDirectReference
|
|
386
|
+
) -> ViewDirectReference:
|
|
387
|
+
"""Normalize through reference to ViewDirectReference for consistent processing.
|
|
388
|
+
|
|
389
|
+
When a reverse direct relation uses a ContainerDirectReference, we convert it to
|
|
390
|
+
a ViewDirectReference using the source view. This enables consistent handling
|
|
391
|
+
in validators regardless of how the 'through' was originally specified.
|
|
392
|
+
"""
|
|
393
|
+
if isinstance(through, ContainerDirectReference):
|
|
394
|
+
return ViewDirectReference(source=source_view_ref, identifier=through.identifier)
|
|
395
|
+
return through
|
|
396
|
+
|
|
397
|
+
@cached_property
|
|
398
|
+
def resolved_reverse_direct_relations(self) -> list[ResolvedReverseDirectRelation]:
|
|
399
|
+
"""Get all reverse direct relations with their resolved context.
|
|
400
|
+
|
|
401
|
+
This property traverses from reverse direct relation → source view → container,
|
|
402
|
+
resolving all references along the way.
|
|
403
|
+
|
|
404
|
+
Includes reverse direct relations where view-level resolution succeeded:
|
|
405
|
+
- The source view exists and can be expanded
|
|
406
|
+
- The through property exists in the source view
|
|
407
|
+
- The through property is a ViewCorePropertyRequest (direct relation)
|
|
408
|
+
|
|
409
|
+
Container-level fields may be None if resolution failed at that level.
|
|
410
|
+
"""
|
|
411
|
+
result: list[ResolvedReverseDirectRelation] = []
|
|
412
|
+
|
|
413
|
+
for (target_view_ref, reverse_prop_id), (
|
|
414
|
+
source_view_ref,
|
|
415
|
+
through,
|
|
416
|
+
) in self.reverse_to_direct_mapping.items():
|
|
417
|
+
through_normalized = self.normalize_through_reference(source_view_ref, through)
|
|
418
|
+
|
|
419
|
+
# Get expanded source view to include inherited properties
|
|
420
|
+
source_view_expanded = self.expand_view_properties(source_view_ref)
|
|
421
|
+
if not source_view_expanded or not source_view_expanded.properties:
|
|
422
|
+
continue
|
|
423
|
+
|
|
424
|
+
if through_normalized.identifier not in source_view_expanded.properties:
|
|
425
|
+
continue
|
|
426
|
+
|
|
427
|
+
source_property = source_view_expanded.properties[through_normalized.identifier]
|
|
428
|
+
|
|
429
|
+
# Must be a core property (direct relation)
|
|
430
|
+
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
431
|
+
continue
|
|
432
|
+
|
|
433
|
+
container_ref = source_property.container
|
|
434
|
+
container_property_id = source_property.container_property_identifier
|
|
435
|
+
|
|
436
|
+
# Resolve container - may be None if missing
|
|
437
|
+
container = self.select_container(container_ref, container_property_id)
|
|
438
|
+
|
|
439
|
+
# Resolve container property - may be None if container missing or property missing
|
|
440
|
+
container_property = None
|
|
441
|
+
if container and container.properties:
|
|
442
|
+
container_property = container.properties.get(container_property_id)
|
|
443
|
+
|
|
444
|
+
result.append(
|
|
445
|
+
ResolvedReverseDirectRelation(
|
|
446
|
+
reverse_view_ref=target_view_ref,
|
|
447
|
+
reverse_property_id=reverse_prop_id,
|
|
448
|
+
direct_view_ref=source_view_ref,
|
|
449
|
+
through_property_id=through_normalized.identifier,
|
|
450
|
+
direct_property=source_property,
|
|
451
|
+
container_ref=container_ref,
|
|
452
|
+
container_property_id=container_property_id,
|
|
453
|
+
container=container,
|
|
454
|
+
container_property=container_property,
|
|
455
|
+
)
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
return result
|
|
459
|
+
|
|
368
460
|
@property
|
|
369
461
|
def connection_end_node_types(self) -> dict[tuple[ViewReference, str], ViewReference | None]:
|
|
370
462
|
"""Get a mapping of view references to their corresponding ViewRequest objects."""
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
|
+
from typing import cast
|
|
2
3
|
|
|
3
4
|
from pydantic import Field, ValidationInfo, field_validator
|
|
4
|
-
from pyparsing import cast
|
|
5
5
|
|
|
6
6
|
from cognite.neat._data_model.models.entities import ConceptEntity
|
|
7
7
|
from cognite.neat._data_model.models.entities._constants import PREFIX_PATTERN, SUFFIX_PATTERN, VERSION_PATTERN
|
|
@@ -39,6 +39,7 @@ from ._limits import (
|
|
|
39
39
|
from ._orchestrator import DmsDataModelRulesOrchestrator
|
|
40
40
|
from ._performance import (
|
|
41
41
|
MissingRequiresConstraint,
|
|
42
|
+
MissingReverseDirectRelationTargetIndex,
|
|
42
43
|
SuboptimalRequiresConstraint,
|
|
43
44
|
UnresolvableQueryPerformance,
|
|
44
45
|
)
|
|
@@ -60,6 +61,7 @@ __all__ = [
|
|
|
60
61
|
"ExternalContainerPropertyDoesNotExist",
|
|
61
62
|
"ImplementedViewNotExisting",
|
|
62
63
|
"MissingRequiresConstraint",
|
|
64
|
+
"MissingReverseDirectRelationTargetIndex",
|
|
63
65
|
"RequiredContainerDoesNotExist",
|
|
64
66
|
"RequiresConstraintCycle",
|
|
65
67
|
"ReverseConnectionContainerMissing",
|
|
@@ -4,7 +4,6 @@ from dataclasses import dataclass
|
|
|
4
4
|
|
|
5
5
|
from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
|
|
6
6
|
from cognite.neat._data_model.models.dms._references import (
|
|
7
|
-
ContainerDirectReference,
|
|
8
7
|
ViewDirectReference,
|
|
9
8
|
ViewReference,
|
|
10
9
|
)
|
|
@@ -119,15 +118,6 @@ class ReverseConnectionContext:
|
|
|
119
118
|
source_view_ref: ViewReference
|
|
120
119
|
|
|
121
120
|
|
|
122
|
-
def _normalize_through_reference(
|
|
123
|
-
source_view_ref: ViewReference, through: ContainerDirectReference | ViewDirectReference
|
|
124
|
-
) -> ViewDirectReference:
|
|
125
|
-
"""Normalize through reference to ViewDirectReference for consistent processing."""
|
|
126
|
-
if isinstance(through, ContainerDirectReference):
|
|
127
|
-
return ViewDirectReference(source=source_view_ref, identifier=through.identifier)
|
|
128
|
-
return through
|
|
129
|
-
|
|
130
|
-
|
|
131
121
|
class ReverseConnectionSourceViewMissing(DataModelRule):
|
|
132
122
|
"""Validates that source view referenced in reverse connection exist.
|
|
133
123
|
|
|
@@ -153,7 +143,7 @@ class ReverseConnectionSourceViewMissing(DataModelRule):
|
|
|
153
143
|
source_view_ref,
|
|
154
144
|
through,
|
|
155
145
|
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
156
|
-
through =
|
|
146
|
+
through = self.validation_resources.normalize_through_reference(source_view_ref, through)
|
|
157
147
|
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
158
148
|
|
|
159
149
|
if not source_view:
|
|
@@ -198,7 +188,7 @@ class ReverseConnectionSourcePropertyMissing(DataModelRule):
|
|
|
198
188
|
source_view_ref,
|
|
199
189
|
through,
|
|
200
190
|
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
201
|
-
through =
|
|
191
|
+
through = self.validation_resources.normalize_through_reference(source_view_ref, through)
|
|
202
192
|
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
203
193
|
|
|
204
194
|
if not source_view:
|
|
@@ -250,7 +240,7 @@ class ReverseConnectionSourcePropertyWrongType(DataModelRule):
|
|
|
250
240
|
source_view_ref,
|
|
251
241
|
through,
|
|
252
242
|
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
253
|
-
through =
|
|
243
|
+
through = self.validation_resources.normalize_through_reference(source_view_ref, through)
|
|
254
244
|
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
255
245
|
|
|
256
246
|
if not source_view:
|
|
@@ -301,40 +291,16 @@ class ReverseConnectionContainerMissing(DataModelRule):
|
|
|
301
291
|
def validate(self) -> list[ConsistencyError]:
|
|
302
292
|
errors: list[ConsistencyError] = []
|
|
303
293
|
|
|
304
|
-
for
|
|
305
|
-
|
|
306
|
-
through,
|
|
307
|
-
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
308
|
-
through = _normalize_through_reference(source_view_ref, through)
|
|
309
|
-
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
310
|
-
|
|
311
|
-
if not source_view:
|
|
312
|
-
continue # Handled by ReverseConnectionSourceViewMissing
|
|
313
|
-
|
|
314
|
-
if not (source_view_expanded := self.validation_resources.expand_view_properties(source_view_ref)):
|
|
315
|
-
raise RuntimeError(f"{type(self).__name__}: View {source_view_ref!s} not found. This is a bug in NEAT.")
|
|
316
|
-
|
|
317
|
-
if not source_view_expanded.properties or through.identifier not in source_view_expanded.properties:
|
|
318
|
-
continue # Handled by ReverseConnectionSourcePropertyMissing
|
|
319
|
-
|
|
320
|
-
source_property = source_view_expanded.properties[through.identifier]
|
|
321
|
-
|
|
322
|
-
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
323
|
-
continue # Handled by ReverseConnectionSourcePropertyWrongType
|
|
324
|
-
|
|
325
|
-
container_ref = source_property.container
|
|
326
|
-
container_property_id = source_property.container_property_identifier
|
|
327
|
-
|
|
328
|
-
source_container = self.validation_resources.select_container(container_ref, container_property_id)
|
|
329
|
-
if not source_container:
|
|
294
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
295
|
+
if resolved.container is None:
|
|
330
296
|
errors.append(
|
|
331
297
|
ConsistencyError(
|
|
332
298
|
message=(
|
|
333
|
-
f"Container {container_ref!s} is missing in both the data model and CDF. "
|
|
334
|
-
f"This container is required by view {
|
|
335
|
-
f" property '{
|
|
336
|
-
f"which configures the reverse connection '{
|
|
337
|
-
f" in target view {
|
|
299
|
+
f"Container {resolved.container_ref!s} is missing in both the data model and CDF. "
|
|
300
|
+
f"This container is required by view {resolved.direct_view_ref!s}"
|
|
301
|
+
f" property '{resolved.through_property_id}', "
|
|
302
|
+
f"which configures the reverse connection '{resolved.reverse_property_id}'"
|
|
303
|
+
f" in target view {resolved.reverse_view_ref!s}."
|
|
338
304
|
),
|
|
339
305
|
fix="Define the missing container",
|
|
340
306
|
code=self.code,
|
|
@@ -366,43 +332,18 @@ class ReverseConnectionContainerPropertyMissing(DataModelRule):
|
|
|
366
332
|
def validate(self) -> list[ConsistencyError]:
|
|
367
333
|
errors: list[ConsistencyError] = []
|
|
368
334
|
|
|
369
|
-
for
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
373
|
-
through = _normalize_through_reference(source_view_ref, through)
|
|
374
|
-
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
375
|
-
|
|
376
|
-
if not source_view:
|
|
377
|
-
continue # Handled by ReverseConnectionSourceViewMissing
|
|
378
|
-
|
|
379
|
-
if not (source_view_expanded := self.validation_resources.expand_view_properties(source_view_ref)):
|
|
380
|
-
raise RuntimeError(f"{type(self).__name__}: View {source_view_ref!s} not found. This is a bug in NEAT.")
|
|
381
|
-
|
|
382
|
-
if not source_view_expanded.properties or through.identifier not in source_view_expanded.properties:
|
|
383
|
-
continue # Handled by ReverseConnectionSourcePropertyMissing
|
|
384
|
-
|
|
385
|
-
source_property = source_view_expanded.properties[through.identifier]
|
|
386
|
-
|
|
387
|
-
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
388
|
-
continue # Handled by ReverseConnectionSourcePropertyWrongType
|
|
389
|
-
|
|
390
|
-
container_ref = source_property.container
|
|
391
|
-
container_property_id = source_property.container_property_identifier
|
|
392
|
-
|
|
393
|
-
source_container = self.validation_resources.select_container(container_ref, container_property_id)
|
|
394
|
-
if not source_container:
|
|
395
|
-
continue # Handled by ReverseConnectionContainerMissing
|
|
396
|
-
|
|
397
|
-
if not source_container.properties or container_property_id not in source_container.properties:
|
|
335
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
336
|
+
# Container must exist but property is missing
|
|
337
|
+
if resolved.container is not None and resolved.container_property is None:
|
|
398
338
|
errors.append(
|
|
399
339
|
ConsistencyError(
|
|
400
340
|
message=(
|
|
401
|
-
f"Container {container_ref!s} is missing
|
|
402
|
-
f"
|
|
403
|
-
f" property
|
|
404
|
-
f"
|
|
405
|
-
f"
|
|
341
|
+
f"Container {resolved.container_ref!s} is missing "
|
|
342
|
+
f"property '{resolved.container_property_id}'. "
|
|
343
|
+
f"This property is required by the source view {resolved.direct_view_ref!s}"
|
|
344
|
+
f" property '{resolved.through_property_id}', "
|
|
345
|
+
f"which configures the reverse connection '{resolved.reverse_property_id}' "
|
|
346
|
+
f"in target view {resolved.reverse_view_ref!s}."
|
|
406
347
|
),
|
|
407
348
|
fix="Add the missing property to the container",
|
|
408
349
|
code=self.code,
|
|
@@ -434,46 +375,21 @@ class ReverseConnectionContainerPropertyWrongType(DataModelRule):
|
|
|
434
375
|
def validate(self) -> list[ConsistencyError]:
|
|
435
376
|
errors: list[ConsistencyError] = []
|
|
436
377
|
|
|
437
|
-
for
|
|
438
|
-
|
|
439
|
-
through,
|
|
440
|
-
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
441
|
-
through = _normalize_through_reference(source_view_ref, through)
|
|
442
|
-
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
443
|
-
|
|
444
|
-
if not source_view:
|
|
445
|
-
continue # Handled by ReverseConnectionSourceViewMissing
|
|
446
|
-
|
|
447
|
-
if not (source_view_expanded := self.validation_resources.expand_view_properties(source_view_ref)):
|
|
448
|
-
raise RuntimeError(f"{type(self).__name__}: View {source_view_ref!s} not found. This is a bug in NEAT.")
|
|
449
|
-
|
|
450
|
-
if not source_view_expanded.properties or through.identifier not in source_view_expanded.properties:
|
|
451
|
-
continue # Handled by ReverseConnectionSourcePropertyMissing
|
|
452
|
-
|
|
453
|
-
source_property = source_view_expanded.properties[through.identifier]
|
|
454
|
-
|
|
455
|
-
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
456
|
-
continue # Handled by ReverseConnectionSourcePropertyWrongType
|
|
457
|
-
|
|
458
|
-
container_ref = source_property.container
|
|
459
|
-
container_property_id = source_property.container_property_identifier
|
|
460
|
-
|
|
461
|
-
source_container = self.validation_resources.select_container(container_ref, container_property_id)
|
|
462
|
-
if not source_container or not source_container.properties:
|
|
463
|
-
continue # Handled by other validators
|
|
464
|
-
|
|
465
|
-
container_property = source_container.properties.get(container_property_id)
|
|
466
|
-
if not container_property:
|
|
378
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
379
|
+
if not resolved.container_property:
|
|
467
380
|
continue # Handled by ReverseConnectionContainerPropertyMissing
|
|
468
381
|
|
|
469
|
-
if not isinstance(container_property.type, DirectNodeRelation):
|
|
382
|
+
if not isinstance(resolved.container_property.type, DirectNodeRelation):
|
|
470
383
|
errors.append(
|
|
471
384
|
ConsistencyError(
|
|
472
385
|
message=(
|
|
473
|
-
f"Container property '{container_property_id}'
|
|
474
|
-
f"
|
|
475
|
-
f"
|
|
476
|
-
f"
|
|
386
|
+
f"Container property '{resolved.container_property_id}' "
|
|
387
|
+
f"in container {resolved.container_ref!s} "
|
|
388
|
+
f"must be a direct connection, but found type '{resolved.container_property.type!s}'. "
|
|
389
|
+
f"This property is used by source view {resolved.direct_view_ref!s} "
|
|
390
|
+
f"property '{resolved.through_property_id}' "
|
|
391
|
+
f"to configure reverse connection '{resolved.reverse_property_id}' "
|
|
392
|
+
f"in target view {resolved.reverse_view_ref!s}."
|
|
477
393
|
),
|
|
478
394
|
fix="Change container property type to be a direct connection",
|
|
479
395
|
code=self.code,
|
|
@@ -504,39 +420,17 @@ class ReverseConnectionTargetMissing(DataModelRule):
|
|
|
504
420
|
def validate(self) -> list[Recommendation]:
|
|
505
421
|
recommendations: list[Recommendation] = []
|
|
506
422
|
|
|
507
|
-
for
|
|
508
|
-
|
|
509
|
-
through,
|
|
510
|
-
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
511
|
-
through = _normalize_through_reference(source_view_ref, through)
|
|
512
|
-
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
513
|
-
|
|
514
|
-
if not source_view:
|
|
515
|
-
continue # Handled by ReverseConnectionSourceViewMissing
|
|
516
|
-
|
|
517
|
-
if not (source_view_expanded := self.validation_resources.expand_view_properties(source_view_ref)):
|
|
518
|
-
raise RuntimeError(f"{type(self).__name__}: View {source_view_ref!s} not found. This is a bug in NEAT.")
|
|
519
|
-
|
|
520
|
-
if not source_view_expanded.properties or through.identifier not in source_view_expanded.properties:
|
|
521
|
-
continue # Handled by ReverseConnectionSourcePropertyMissing
|
|
522
|
-
|
|
523
|
-
source_property = source_view_expanded.properties[through.identifier]
|
|
524
|
-
|
|
525
|
-
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
526
|
-
continue # Handled by ReverseConnectionSourcePropertyWrongType
|
|
527
|
-
|
|
528
|
-
actual_target_view = source_property.source
|
|
529
|
-
|
|
530
|
-
if not actual_target_view:
|
|
423
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
424
|
+
if not resolved.direct_property.source:
|
|
531
425
|
recommendations.append(
|
|
532
426
|
Recommendation(
|
|
533
427
|
message=(
|
|
534
|
-
f"Source view {
|
|
428
|
+
f"Source view {resolved.direct_view_ref!s} property '{resolved.through_property_id}' "
|
|
535
429
|
f"has no target view specified (value type is None). "
|
|
536
|
-
f"This property is used for reverse connection '{
|
|
537
|
-
f"in target view {
|
|
430
|
+
f"This property is used for reverse connection '{resolved.reverse_property_id}' "
|
|
431
|
+
f"in target view {resolved.reverse_view_ref!s}. "
|
|
538
432
|
f"While this works as a hack for multi-value relations in CDF Search, "
|
|
539
|
-
f"it's recommended to explicitly define the target view as {
|
|
433
|
+
f"it's recommended to explicitly define the target view as {resolved.reverse_view_ref!s}."
|
|
540
434
|
),
|
|
541
435
|
fix="Set the property's value type to the target view for better clarity",
|
|
542
436
|
code=self.code,
|
|
@@ -568,41 +462,24 @@ class ReverseConnectionPointsToAncestor(DataModelRule):
|
|
|
568
462
|
def validate(self) -> list[Recommendation]:
|
|
569
463
|
recommendations: list[Recommendation] = []
|
|
570
464
|
|
|
571
|
-
for
|
|
572
|
-
|
|
573
|
-
through,
|
|
574
|
-
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
575
|
-
through = _normalize_through_reference(source_view_ref, through)
|
|
576
|
-
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
577
|
-
|
|
578
|
-
if not source_view:
|
|
579
|
-
continue # Handled by ReverseConnectionSourceViewMissing
|
|
580
|
-
|
|
581
|
-
if not (source_view_expanded := self.validation_resources.expand_view_properties(source_view_ref)):
|
|
582
|
-
raise RuntimeError(f"{type(self).__name__}: View {source_view_ref!s} not found. This is a bug in NEAT.")
|
|
583
|
-
|
|
584
|
-
if not source_view_expanded.properties or through.identifier not in source_view_expanded.properties:
|
|
585
|
-
continue # Handled by ReverseConnectionSourcePropertyMissing
|
|
586
|
-
|
|
587
|
-
source_property = source_view_expanded.properties[through.identifier]
|
|
588
|
-
|
|
589
|
-
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
590
|
-
continue # Handled by other validators
|
|
591
|
-
|
|
592
|
-
actual_target_view = source_property.source
|
|
465
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
466
|
+
actual_target_view = resolved.direct_property.source
|
|
593
467
|
|
|
594
468
|
if not actual_target_view:
|
|
595
469
|
continue # Handled by ReverseConnectionTargetMissing
|
|
596
470
|
|
|
597
|
-
if self.validation_resources.is_ancestor(
|
|
471
|
+
if self.validation_resources.is_ancestor(resolved.reverse_view_ref, actual_target_view):
|
|
598
472
|
recommendations.append(
|
|
599
473
|
Recommendation(
|
|
600
474
|
message=(
|
|
601
|
-
f"The direct connection property '{
|
|
602
|
-
f"
|
|
603
|
-
f"
|
|
475
|
+
f"The direct connection property '{resolved.through_property_id}' "
|
|
476
|
+
f"in view {resolved.direct_view_ref!s} "
|
|
477
|
+
f"configures the reverse connection '{resolved.reverse_property_id}' "
|
|
478
|
+
f"in {resolved.reverse_view_ref!s}. "
|
|
479
|
+
f"Therefore, it is expected that '{resolved.through_property_id}' "
|
|
480
|
+
f"points to {resolved.reverse_view_ref!s}. "
|
|
604
481
|
f"However, it currently points to {actual_target_view!s}, which is an ancestor of "
|
|
605
|
-
f"{
|
|
482
|
+
f"{resolved.reverse_view_ref!s}. "
|
|
606
483
|
"While this will allow for model to be valid, it can be a source of confusion and mistakes."
|
|
607
484
|
),
|
|
608
485
|
fix="Update the direct connection property to point to the target view instead of its ancestor",
|
|
@@ -635,42 +512,24 @@ class ReverseConnectionTargetMismatch(DataModelRule):
|
|
|
635
512
|
def validate(self) -> list[Recommendation]:
|
|
636
513
|
recommendations: list[Recommendation] = []
|
|
637
514
|
|
|
638
|
-
for
|
|
639
|
-
|
|
640
|
-
through,
|
|
641
|
-
) in self.validation_resources.reverse_to_direct_mapping.items():
|
|
642
|
-
through = _normalize_through_reference(source_view_ref, through)
|
|
643
|
-
source_view = self.validation_resources.select_view(source_view_ref, through.identifier)
|
|
644
|
-
|
|
645
|
-
if not source_view:
|
|
646
|
-
continue # Handled by ReverseConnectionSourceViewMissing
|
|
647
|
-
|
|
648
|
-
if not (source_view_expanded := self.validation_resources.expand_view_properties(source_view_ref)):
|
|
649
|
-
raise RuntimeError(f"{type(self).__name__}: View {source_view_ref!s} not found. This is a bug in NEAT.")
|
|
650
|
-
|
|
651
|
-
if not source_view_expanded.properties or through.identifier not in source_view_expanded.properties:
|
|
652
|
-
continue # Handled by ReverseConnectionSourcePropertyMissing
|
|
653
|
-
|
|
654
|
-
source_property = source_view_expanded.properties[through.identifier]
|
|
655
|
-
|
|
656
|
-
if not isinstance(source_property, ViewCorePropertyRequest):
|
|
657
|
-
continue # Handled by other validators
|
|
658
|
-
|
|
659
|
-
actual_target_view = source_property.source
|
|
515
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
516
|
+
actual_target_view = resolved.direct_property.source
|
|
660
517
|
|
|
661
518
|
if not actual_target_view:
|
|
662
519
|
continue # Handled by ReverseConnectionTargetMissing
|
|
663
520
|
|
|
664
|
-
if self.validation_resources.is_ancestor(
|
|
665
|
-
continue # Handled by
|
|
521
|
+
if self.validation_resources.is_ancestor(resolved.reverse_view_ref, actual_target_view):
|
|
522
|
+
continue # Handled by ReverseConnectionPointsToAncestor
|
|
666
523
|
|
|
667
|
-
if actual_target_view !=
|
|
524
|
+
if actual_target_view != resolved.reverse_view_ref:
|
|
668
525
|
recommendations.append(
|
|
669
526
|
Recommendation(
|
|
670
527
|
message=(
|
|
671
|
-
f"The reverse connection '{
|
|
672
|
-
f"
|
|
673
|
-
f"
|
|
528
|
+
f"The reverse connection '{resolved.reverse_property_id}' "
|
|
529
|
+
f"in view {resolved.reverse_view_ref!s} "
|
|
530
|
+
f"expects its corresponding direct connection in view {resolved.direct_view_ref!s} "
|
|
531
|
+
f"(property '{resolved.through_property_id}') "
|
|
532
|
+
f"to point back to {resolved.reverse_view_ref!s}, "
|
|
674
533
|
f"but it actually points to {actual_target_view!s}."
|
|
675
534
|
),
|
|
676
535
|
fix="Update the direct connection property to point back to the correct target view",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Validators for checking containers in the data model."""
|
|
2
2
|
|
|
3
|
-
from
|
|
3
|
+
from typing import cast
|
|
4
4
|
|
|
5
5
|
from cognite.neat._data_model.models.dms._constraints import Constraint, RequiresConstraintDefinition
|
|
6
6
|
from cognite.neat._data_model.models.dms._view_property import ViewCorePropertyRequest
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
from cognite.neat._data_model._analysis import RequiresChangeStatus
|
|
4
4
|
from cognite.neat._data_model._constants import COGNITE_SPACES
|
|
5
|
+
from cognite.neat._data_model.models.dms._data_types import DirectNodeRelation
|
|
6
|
+
from cognite.neat._data_model.models.dms._indexes import BtreeIndex
|
|
5
7
|
from cognite.neat._data_model.rules.dms._base import DataModelRule
|
|
6
8
|
from cognite.neat._issues import Recommendation
|
|
7
9
|
|
|
@@ -193,3 +195,76 @@ class UnresolvableQueryPerformance(DataModelRule):
|
|
|
193
195
|
)
|
|
194
196
|
|
|
195
197
|
return recommendations
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class MissingReverseDirectRelationTargetIndex(DataModelRule):
|
|
201
|
+
"""
|
|
202
|
+
Recommends adding a cursorable index on direct relation properties that are
|
|
203
|
+
targets of reverse direct relations for query performance.
|
|
204
|
+
|
|
205
|
+
## What it does
|
|
206
|
+
Identifies direct relation properties that are referenced by reverse direct relations
|
|
207
|
+
but lack a cursorable B-tree index. When querying through a reverse direct relation,
|
|
208
|
+
CDF needs to look up nodes that have the direct relation pointing to the
|
|
209
|
+
source nodes. Without an index, this requires scanning many nodes inefficiently.
|
|
210
|
+
|
|
211
|
+
## Why is this important?
|
|
212
|
+
Traversing a reverse direct relation (inwards direction) requires an index on the
|
|
213
|
+
target direct relation property. Without this index, queries will be inefficient,
|
|
214
|
+
potentially leading to timeouts over time, as they won't scale well with data volume.
|
|
215
|
+
|
|
216
|
+
The exception is when the target direct relation is a list property. In that case,
|
|
217
|
+
this validator will not flag them, as reverse direct relations targeting lists of
|
|
218
|
+
direct relations needs to be traversed using the `instances/search` endpoint instead,
|
|
219
|
+
which does not directly benefit from adding indexes to container properties.
|
|
220
|
+
|
|
221
|
+
## Example
|
|
222
|
+
View `WindFarm` has a reverse property `turbines` through `WindTurbine.windFarm`.
|
|
223
|
+
Container `WindTurbine` should have a cursorable B-tree index on the `windFarm`
|
|
224
|
+
property to enable efficient traversal from WindFarm to its turbines.
|
|
225
|
+
"""
|
|
226
|
+
|
|
227
|
+
code = f"{BASE_CODE}-004"
|
|
228
|
+
issue_type = Recommendation
|
|
229
|
+
alpha = True
|
|
230
|
+
|
|
231
|
+
def validate(self) -> list[Recommendation]:
|
|
232
|
+
recommendations: list[Recommendation] = []
|
|
233
|
+
|
|
234
|
+
for resolved in self.validation_resources.resolved_reverse_direct_relations:
|
|
235
|
+
# Skip if container or container property couldn't be resolved
|
|
236
|
+
if not resolved.container or not resolved.container_property:
|
|
237
|
+
continue
|
|
238
|
+
|
|
239
|
+
# Must be a DirectNodeRelation type (other types handled by ReverseConnectionContainerPropertyWrongType)
|
|
240
|
+
if not isinstance(resolved.container_property.type, DirectNodeRelation):
|
|
241
|
+
continue
|
|
242
|
+
|
|
243
|
+
# Skip if this is a list direct relation - indexes are not supported for list properties
|
|
244
|
+
if resolved.container_property.type.list:
|
|
245
|
+
continue
|
|
246
|
+
|
|
247
|
+
# Skip if there's already a cursorable B-tree index on this property
|
|
248
|
+
if resolved.container.indexes and any(
|
|
249
|
+
isinstance(index, BtreeIndex)
|
|
250
|
+
and index.cursorable
|
|
251
|
+
and resolved.container_property_id in index.properties
|
|
252
|
+
for index in resolved.container.indexes.values()
|
|
253
|
+
):
|
|
254
|
+
continue
|
|
255
|
+
|
|
256
|
+
recommendations.append(
|
|
257
|
+
Recommendation(
|
|
258
|
+
message=(
|
|
259
|
+
f"View '{resolved.reverse_view_ref!s}' has a reverse direct relation "
|
|
260
|
+
f"'{resolved.reverse_property_id}' that points to container "
|
|
261
|
+
f"'{resolved.container_ref!s}' property '{resolved.container_property_id}'. "
|
|
262
|
+
f"Add a cursorable B-tree index on this target container property "
|
|
263
|
+
f"to enable efficient query traversal."
|
|
264
|
+
),
|
|
265
|
+
fix="Add a cursorable B-tree index on the target direct relation property",
|
|
266
|
+
code=self.code,
|
|
267
|
+
)
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
return recommendations
|
cognite/neat/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "1.0.
|
|
1
|
+
__version__ = "1.0.35"
|
|
2
2
|
__engine__ = "^2.0.4"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cognite-neat
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.35
|
|
4
4
|
Summary: Knowledge graph transformation
|
|
5
5
|
Author: Nikola Vasiljevic, Anders Albert
|
|
6
6
|
Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>
|
|
@@ -17,7 +17,7 @@ cognite/neat/_client/statistics_api.py,sha256=3KNVsD2OV-EDFb1eqIrCXfYwjvyCDO80bP
|
|
|
17
17
|
cognite/neat/_client/views_api.py,sha256=YMaw7IaxU4gmixpd_t1u9JK9BHfNerf5DMNinGPCAa0,3692
|
|
18
18
|
cognite/neat/_config.py,sha256=IJS_R-86M4SLxdo644LZ0dE9h0jg2XBL1A4QKWJj0TQ,10160
|
|
19
19
|
cognite/neat/_data_model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
-
cognite/neat/_data_model/_analysis.py,sha256=
|
|
20
|
+
cognite/neat/_data_model/_analysis.py,sha256=VUheUE1xZcfXxOzF9wW6iOmsbWWIg_GB6X6HoD0PYbU,47799
|
|
21
21
|
cognite/neat/_data_model/_constants.py,sha256=pPDtRwBsOq80IkYXRJ0XaRfFOb7CPvxz81IQzcPgAuc,1889
|
|
22
22
|
cognite/neat/_data_model/_identifiers.py,sha256=lDLvMvYDgRNFgk5GmxWzOUunG7M3synAciNjzJI0m_o,1913
|
|
23
23
|
cognite/neat/_data_model/_shared.py,sha256=H0gFqa8tKFNWuvdat5jL6OwySjCw3aQkLPY3wtb9Wrw,1302
|
|
@@ -48,7 +48,7 @@ cognite/neat/_data_model/importers/_table_importer/source.py,sha256=h7u5ur5oetmv
|
|
|
48
48
|
cognite/neat/_data_model/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
49
49
|
cognite/neat/_data_model/models/conceptual/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
50
|
cognite/neat/_data_model/models/conceptual/_base.py,sha256=SFkoBJDM51pqew_isHFJoB20OgfofpwVRnTrg-rKkNY,710
|
|
51
|
-
cognite/neat/_data_model/models/conceptual/_concept.py,sha256=
|
|
51
|
+
cognite/neat/_data_model/models/conceptual/_concept.py,sha256=eQMOb3tJJHvopr3QqP3XkP5GjBO4UXTKoVy_eSEwzEk,2245
|
|
52
52
|
cognite/neat/_data_model/models/conceptual/_data_model.py,sha256=zvJZi0OqOFMviwY84Am3Oz_RmF1g3tOzANFHlrldBuU,1673
|
|
53
53
|
cognite/neat/_data_model/models/conceptual/_properties.py,sha256=CpF37vJYBTLT4DH4ZOu2U-JyWtkb_27V8fw52qiaE_k,4007
|
|
54
54
|
cognite/neat/_data_model/models/conceptual/_property.py,sha256=blSZQxX52zaILAtjUkldPzPeysz7wnG-UGSNU5tacI8,4138
|
|
@@ -81,15 +81,15 @@ cognite/neat/_data_model/rules/cdf/__init__.py,sha256=Bn9_DkkO4OPUYh0CsM3Pf1fD_U
|
|
|
81
81
|
cognite/neat/_data_model/rules/cdf/_base.py,sha256=59fqmFi_Gvzx3swLsIyc2ACkHAzKAKiqNlwwGnW5bDg,128
|
|
82
82
|
cognite/neat/_data_model/rules/cdf/_orchestrator.py,sha256=egzcYUuFDd8Vm7neBpZFofqZBfYjuElAjYwJWAScutc,2250
|
|
83
83
|
cognite/neat/_data_model/rules/cdf/_spaces.py,sha256=Lsfrp2e7yzgRimkD0qynJ5M2S7Zwzplfg_4Pim1voHk,1605
|
|
84
|
-
cognite/neat/_data_model/rules/dms/__init__.py,sha256=
|
|
84
|
+
cognite/neat/_data_model/rules/dms/__init__.py,sha256=LiZNGYClXHrgSbG0K6tk8nYN2B0VX3JlKn_minyCiI8,3042
|
|
85
85
|
cognite/neat/_data_model/rules/dms/_ai_readiness.py,sha256=1KvOk3Q8aQkE8G6oHml7O-SHiTeIBNmyKPWr3hPcCCw,16133
|
|
86
86
|
cognite/neat/_data_model/rules/dms/_base.py,sha256=BnzBELon389lEXJi7dlb9t3SPjiXgVquiI445I7-APU,134
|
|
87
|
-
cognite/neat/_data_model/rules/dms/_connections.py,sha256=
|
|
87
|
+
cognite/neat/_data_model/rules/dms/_connections.py,sha256=yvlcb_-lBILZpMZ9Zcp8R1sC-G6DhCdyqjffwOnlmUI,23416
|
|
88
88
|
cognite/neat/_data_model/rules/dms/_consistency.py,sha256=7u-EDQk08eV4CJoY5nRbrYXs2cbkQkH6TQCcPOyUGI8,2425
|
|
89
|
-
cognite/neat/_data_model/rules/dms/_containers.py,sha256=
|
|
89
|
+
cognite/neat/_data_model/rules/dms/_containers.py,sha256=R67rAlA2V9oMbp0bQEpHu8VF1Yr3f8xXgox7NkQtWk8,10539
|
|
90
90
|
cognite/neat/_data_model/rules/dms/_limits.py,sha256=L6dAsLWVBEpfVFZgw3SvJk1PnEGf4lhJ3SOXjDUsY2Q,14820
|
|
91
91
|
cognite/neat/_data_model/rules/dms/_orchestrator.py,sha256=tTXxLH1yThAeTW_GdDeD-to19faK72c_z8QL9kzgGJc,3069
|
|
92
|
-
cognite/neat/_data_model/rules/dms/_performance.py,sha256=
|
|
92
|
+
cognite/neat/_data_model/rules/dms/_performance.py,sha256=Tp7D5cCEOqjfst9tLKuA9X2F9f4lQLJHEiGojm-rQUI,12229
|
|
93
93
|
cognite/neat/_data_model/rules/dms/_views.py,sha256=Tp4VW2512eGSGgeMRoUFh7hqK2vsZD8ryN_EJ7QJoEg,6335
|
|
94
94
|
cognite/neat/_exceptions.py,sha256=hOjPL1vFNNAZzqAHFB9l9ek-XJEBKcqiaPk0onwLPns,2540
|
|
95
95
|
cognite/neat/_issues.py,sha256=wH1mnkrpBsHUkQMGUHFLUIQWQlfJ_qMfdF7q0d9wNhY,1871
|
|
@@ -333,9 +333,9 @@ cognite/neat/_v0/session/_template.py,sha256=BNcvrW5y7LWzRM1XFxZkfR1Nc7e8UgjBClH
|
|
|
333
333
|
cognite/neat/_v0/session/_to.py,sha256=AnsRSDDdfFyYwSgi0Z-904X7WdLtPfLlR0x1xsu_jAo,19447
|
|
334
334
|
cognite/neat/_v0/session/_wizard.py,sha256=baPJgXAAF3d1bn4nbIzon1gWfJOeS5T43UXRDJEnD3c,1490
|
|
335
335
|
cognite/neat/_v0/session/exceptions.py,sha256=jv52D-SjxGfgqaHR8vnpzo0SOJETIuwbyffSWAxSDJw,3495
|
|
336
|
-
cognite/neat/_version.py,sha256=
|
|
336
|
+
cognite/neat/_version.py,sha256=pfiLq3MLQ7wyaAim2tr-d61GfU9oqrmB9vFVh7JRzNY,45
|
|
337
337
|
cognite/neat/legacy.py,sha256=DMFeLCSBLT2enk-nm1KfX1rKR2DQDpxY-w6ThY0y9c8,421
|
|
338
338
|
cognite/neat/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
339
|
-
cognite_neat-1.0.
|
|
340
|
-
cognite_neat-1.0.
|
|
341
|
-
cognite_neat-1.0.
|
|
339
|
+
cognite_neat-1.0.35.dist-info/WHEEL,sha256=5DEXXimM34_d4Gx1AuF9ysMr1_maoEtGKjaILM3s4w4,80
|
|
340
|
+
cognite_neat-1.0.35.dist-info/METADATA,sha256=ra9SuJzQhAi_Yw6BM6qV-nnNNQ8DUCtCDG2RCe1hVxs,6872
|
|
341
|
+
cognite_neat-1.0.35.dist-info/RECORD,,
|