cognite-neat 0.88.2__py3-none-any.whl → 0.88.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_version.py +1 -1
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/loaders/_base.py +3 -3
- cognite/neat/graph/loaders/_rdf2asset.py +24 -25
- cognite/neat/graph/loaders/_rdf2dms.py +20 -15
- cognite/neat/issues/__init__.py +1 -3
- cognite/neat/issues/_base.py +259 -70
- cognite/neat/issues/errors/__init__.py +72 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +28 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/issues/formatters.py +1 -1
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +10 -2
- cognite/neat/rules/exporters/_base.py +6 -6
- cognite/neat/rules/exporters/_rules2dms.py +18 -11
- cognite/neat/rules/exporters/_rules2excel.py +4 -4
- cognite/neat/rules/exporters/_rules2ontology.py +74 -51
- cognite/neat/rules/exporters/_rules2yaml.py +3 -3
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/_base.py +8 -12
- cognite/neat/rules/importers/_dms2rules.py +21 -24
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +22 -17
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +26 -19
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +1 -1
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +9 -7
- cognite/neat/rules/importers/_rdf/_inference2rules.py +8 -8
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +4 -4
- cognite/neat/rules/importers/_rdf/_shared.py +3 -3
- cognite/neat/rules/importers/_spreadsheet2rules.py +35 -22
- cognite/neat/rules/importers/_yaml2rules.py +23 -22
- cognite/neat/rules/models/_constants.py +2 -1
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +5 -10
- cognite/neat/rules/models/asset/_rules.py +1 -3
- cognite/neat/rules/models/asset/_validation.py +13 -9
- cognite/neat/rules/models/dms/_converter.py +2 -4
- cognite/neat/rules/models/dms/_exporter.py +30 -8
- cognite/neat/rules/models/dms/_rules.py +23 -7
- cognite/neat/rules/models/dms/_schema.py +87 -78
- cognite/neat/rules/models/dms/_validation.py +104 -65
- cognite/neat/rules/models/information/_converter.py +2 -2
- cognite/neat/rules/models/information/_rules.py +7 -8
- cognite/neat/rules/models/information/_validation.py +47 -24
- cognite/neat/rules/transformers/_base.py +15 -0
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -10
- cognite/neat/workflows/steps/lib/current/rules_importer.py +6 -6
- cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/RECORD +78 -84
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/issues/errors/external.py +0 -21
- cognite/neat/issues/errors/properties.py +0 -75
- cognite/neat/issues/errors/resources.py +0 -123
- cognite/neat/issues/neat_warnings/__init__.py +0 -2
- cognite/neat/issues/neat_warnings/identifier.py +0 -27
- cognite/neat/issues/neat_warnings/models.py +0 -22
- cognite/neat/issues/neat_warnings/properties.py +0 -77
- cognite/neat/issues/neat_warnings/resources.py +0 -125
- cognite/neat/rules/issues/__init__.py +0 -22
- cognite/neat/rules/issues/base.py +0 -63
- cognite/neat/rules/issues/dms.py +0 -549
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{issues/errors/schema.py → rules/transformers/__init__.py} +0 -0
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- /cognite/neat/{graph/stores → store}/_provenance.py +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.88.3.dist-info}/entry_points.txt +0 -0
|
@@ -26,17 +26,20 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
26
26
|
from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
|
|
27
27
|
|
|
28
28
|
from cognite.neat.issues import NeatError
|
|
29
|
-
from cognite.neat.issues.errors
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
DirectRelationMissingSourceWarning,
|
|
36
|
-
DuplicatedViewInDataModelError,
|
|
37
|
-
IncompleteSchemaError,
|
|
38
|
-
MissingViewInModelWarning,
|
|
29
|
+
from cognite.neat.issues.errors import (
|
|
30
|
+
NeatYamlError,
|
|
31
|
+
PropertyMappingDuplicatedError,
|
|
32
|
+
PropertyNotFoundError,
|
|
33
|
+
ResourceDuplicatedError,
|
|
34
|
+
ResourceNotFoundError,
|
|
39
35
|
)
|
|
36
|
+
from cognite.neat.issues.warnings import (
|
|
37
|
+
FileTypeUnexpectedWarning,
|
|
38
|
+
ResourceNotFoundWarning,
|
|
39
|
+
ResourceRetrievalWarning,
|
|
40
|
+
ResourcesDuplicatedWarning,
|
|
41
|
+
)
|
|
42
|
+
from cognite.neat.issues.warnings.user_modeling import DirectRelationMissingSourceWarning
|
|
40
43
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
|
|
41
44
|
from cognite.neat.utils.cdf.data_classes import (
|
|
42
45
|
CogniteResourceDict,
|
|
@@ -93,11 +96,11 @@ class DMSSchema:
|
|
|
93
96
|
directly_referenced_containers = view_by_id[view_id].referenced_containers()
|
|
94
97
|
inherited_referenced_containers = set()
|
|
95
98
|
|
|
96
|
-
for
|
|
97
|
-
if implemented_view := view_by_id.get(
|
|
99
|
+
for parent_id in view_inheritance:
|
|
100
|
+
if implemented_view := view_by_id.get(parent_id):
|
|
98
101
|
inherited_referenced_containers |= implemented_view.referenced_containers()
|
|
99
102
|
else:
|
|
100
|
-
raise
|
|
103
|
+
raise ResourceNotFoundError(parent_id, "view", view_id, "view")
|
|
101
104
|
|
|
102
105
|
return directly_referenced_containers | inherited_referenced_containers
|
|
103
106
|
|
|
@@ -164,10 +167,14 @@ class DMSSchema:
|
|
|
164
167
|
connection_referenced_view_ids |= cls._connection_references(view)
|
|
165
168
|
connection_referenced_view_ids = connection_referenced_view_ids - existing_view_ids
|
|
166
169
|
if connection_referenced_view_ids:
|
|
167
|
-
|
|
170
|
+
for view_id in connection_referenced_view_ids:
|
|
171
|
+
warnings.warn(
|
|
172
|
+
ResourceNotFoundWarning(view_id, "view", data_model_write.as_id(), "data model"),
|
|
173
|
+
stacklevel=2,
|
|
174
|
+
)
|
|
168
175
|
connection_referenced_views = view_loader.retrieve(list(connection_referenced_view_ids))
|
|
169
176
|
if failed := connection_referenced_view_ids - set(connection_referenced_views.as_ids()):
|
|
170
|
-
warnings.warn(
|
|
177
|
+
warnings.warn(ResourceRetrievalWarning(frozenset(failed), "view"), stacklevel=2)
|
|
171
178
|
views.extend(connection_referenced_views)
|
|
172
179
|
|
|
173
180
|
# We need to include parent views in the schema to make sure that the schema is valid.
|
|
@@ -262,10 +269,11 @@ class DMSSchema:
|
|
|
262
269
|
data.setdefault(attr_name, [])
|
|
263
270
|
context.setdefault(attr_name, [])
|
|
264
271
|
try:
|
|
265
|
-
# Using CSafeLoader over safe_load for ~10x speedup
|
|
266
272
|
loaded = yaml.safe_load(yaml_file.read_text())
|
|
267
273
|
except Exception as e:
|
|
268
|
-
warnings.warn(
|
|
274
|
+
warnings.warn(
|
|
275
|
+
FileTypeUnexpectedWarning(yaml_file, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
|
|
276
|
+
)
|
|
269
277
|
continue
|
|
270
278
|
|
|
271
279
|
if isinstance(loaded, list):
|
|
@@ -355,7 +363,9 @@ class DMSSchema:
|
|
|
355
363
|
try:
|
|
356
364
|
loaded = yaml.safe_load(zip_ref.read(file_info).decode())
|
|
357
365
|
except Exception as e:
|
|
358
|
-
warnings.warn(
|
|
366
|
+
warnings.warn(
|
|
367
|
+
FileTypeUnexpectedWarning(filename, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
|
|
368
|
+
)
|
|
359
369
|
continue
|
|
360
370
|
if isinstance(loaded, list):
|
|
361
371
|
data[attr_name].extend(loaded)
|
|
@@ -411,11 +421,9 @@ class DMSSchema:
|
|
|
411
421
|
try:
|
|
412
422
|
data_dict = yaml.safe_load(data)
|
|
413
423
|
except Exception as e:
|
|
414
|
-
raise
|
|
424
|
+
raise NeatYamlError(str(e)) from None
|
|
415
425
|
if not isinstance(data_dict, dict) and all(isinstance(v, list) for v in data_dict.values()):
|
|
416
|
-
raise
|
|
417
|
-
"dict[str, list[Any]]", f"Invalid data structure: {type(data)}"
|
|
418
|
-
).as_exception() from None
|
|
426
|
+
raise NeatYamlError(f"Invalid data structure: {type(data)}", "dict[str, list[Any]]") from None
|
|
419
427
|
else:
|
|
420
428
|
data_dict = data
|
|
421
429
|
loaded: dict[str, Any] = {}
|
|
@@ -423,20 +431,32 @@ class DMSSchema:
|
|
|
423
431
|
if items := data_dict.get(attr.name) or data_dict.get(to_camel(attr.name)):
|
|
424
432
|
if attr.name == "data_model":
|
|
425
433
|
if isinstance(items, list) and len(items) > 1:
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
434
|
+
try:
|
|
435
|
+
data_model_ids = [dm.DataModelId.load(item) for item in items]
|
|
436
|
+
except Exception as e:
|
|
437
|
+
data_model_file = context.get(attr.name, [Path("UNKNOWN")])[0]
|
|
438
|
+
warnings.warn(
|
|
439
|
+
FileTypeUnexpectedWarning(
|
|
440
|
+
data_model_file, frozenset([dm.DataModelApply.__name__]), str(e)
|
|
441
|
+
),
|
|
442
|
+
stacklevel=2,
|
|
443
|
+
)
|
|
444
|
+
else:
|
|
445
|
+
warnings.warn(
|
|
446
|
+
ResourcesDuplicatedWarning(
|
|
447
|
+
frozenset(data_model_ids),
|
|
448
|
+
"data model",
|
|
449
|
+
"Will use the first DataModel.",
|
|
450
|
+
),
|
|
451
|
+
stacklevel=2,
|
|
452
|
+
)
|
|
433
453
|
item = items[0] if isinstance(items, list) else items
|
|
434
454
|
try:
|
|
435
455
|
loaded[attr.name] = dm.DataModelApply.load(item)
|
|
436
456
|
except Exception as e:
|
|
437
457
|
data_model_file = context.get(attr.name, [Path("UNKNOWN")])[0]
|
|
438
458
|
warnings.warn(
|
|
439
|
-
|
|
459
|
+
FileTypeUnexpectedWarning(data_model_file, frozenset([dm.DataModelApply.__name__]), str(e)),
|
|
440
460
|
stacklevel=2,
|
|
441
461
|
)
|
|
442
462
|
else:
|
|
@@ -453,7 +473,7 @@ class DMSSchema:
|
|
|
453
473
|
resources = attr.type([])
|
|
454
474
|
if not hasattr(attr.type, "_RESOURCE"):
|
|
455
475
|
warnings.warn(
|
|
456
|
-
|
|
476
|
+
FileTypeUnexpectedWarning(Path("UNKNOWN"), frozenset([attr.type.__name__]), trigger_error), stacklevel=2
|
|
457
477
|
)
|
|
458
478
|
return resources
|
|
459
479
|
# Fallback to load individual resources.
|
|
@@ -467,7 +487,9 @@ class DMSSchema:
|
|
|
467
487
|
except IndexError:
|
|
468
488
|
filepath = Path("UNKNOWN")
|
|
469
489
|
# We use repr(e) instead of str(e) to include the exception type in the warning message
|
|
470
|
-
warnings.warn(
|
|
490
|
+
warnings.warn(
|
|
491
|
+
FileTypeUnexpectedWarning(filepath, frozenset([single_cls.__name__]), repr(e)), stacklevel=2
|
|
492
|
+
)
|
|
471
493
|
else:
|
|
472
494
|
resources.append(loaded_instance)
|
|
473
495
|
return resources
|
|
@@ -529,41 +551,31 @@ class DMSSchema:
|
|
|
529
551
|
for container in self.containers.values():
|
|
530
552
|
if container.space not in defined_spaces:
|
|
531
553
|
errors.add(
|
|
532
|
-
|
|
533
|
-
container.space, "Space", container.as_id(), "Container"
|
|
534
|
-
)
|
|
554
|
+
ResourceNotFoundError[str, dm.ContainerId](container.space, "space", container.as_id(), "container")
|
|
535
555
|
)
|
|
536
556
|
|
|
537
557
|
for view in self.views.values():
|
|
538
558
|
view_id = view.as_id()
|
|
539
559
|
if view.space not in defined_spaces:
|
|
540
|
-
errors.add(
|
|
560
|
+
errors.add(ResourceNotFoundError(view.space, "space", view_id, "view"))
|
|
541
561
|
|
|
542
562
|
for parent in view.implements or []:
|
|
543
563
|
if parent not in defined_views:
|
|
544
|
-
errors.add(
|
|
545
|
-
ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
|
|
546
|
-
parent, "View", view_id, "View", property_name="implements"
|
|
547
|
-
)
|
|
548
|
-
)
|
|
564
|
+
errors.add(PropertyNotFoundError(parent, "view", "implements", view_id, "view"))
|
|
549
565
|
|
|
550
566
|
for prop_name, prop in (view.properties or {}).items():
|
|
551
567
|
if isinstance(prop, dm.MappedPropertyApply):
|
|
552
568
|
ref_container = defined_containers.get(prop.container)
|
|
553
569
|
if ref_container is None:
|
|
554
|
-
errors.add(
|
|
555
|
-
ReferredResourceNotFoundError[dm.ContainerId, dm.ViewId](
|
|
556
|
-
prop.container, "Container", view_id, "View"
|
|
557
|
-
)
|
|
558
|
-
)
|
|
570
|
+
errors.add(ResourceNotFoundError(prop.container, "container", view_id, "view"))
|
|
559
571
|
elif prop.container_property_identifier not in ref_container.properties:
|
|
560
572
|
errors.add(
|
|
561
|
-
|
|
573
|
+
PropertyNotFoundError(
|
|
562
574
|
prop.container,
|
|
563
|
-
"
|
|
575
|
+
"container",
|
|
576
|
+
prop.container_property_identifier,
|
|
564
577
|
view_id,
|
|
565
|
-
"
|
|
566
|
-
property_name=prop.container_property_identifier,
|
|
578
|
+
"view",
|
|
567
579
|
)
|
|
568
580
|
)
|
|
569
581
|
else:
|
|
@@ -571,26 +583,19 @@ class DMSSchema:
|
|
|
571
583
|
|
|
572
584
|
if isinstance(container_property.type, dm.DirectRelation) and prop.source is None:
|
|
573
585
|
warnings.warn(
|
|
574
|
-
DirectRelationMissingSourceWarning(view_id
|
|
586
|
+
DirectRelationMissingSourceWarning(view_id, prop_name),
|
|
587
|
+
stacklevel=2,
|
|
575
588
|
)
|
|
576
589
|
|
|
577
590
|
if isinstance(prop, dm.EdgeConnectionApply) and prop.source not in defined_views:
|
|
578
|
-
errors.add(
|
|
579
|
-
ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
|
|
580
|
-
prop.source, "View", view_id, "View", property_name=prop_name
|
|
581
|
-
)
|
|
582
|
-
)
|
|
591
|
+
errors.add(PropertyNotFoundError(prop.source, "view", prop_name, view_id, "view"))
|
|
583
592
|
|
|
584
593
|
if (
|
|
585
594
|
isinstance(prop, dm.EdgeConnectionApply)
|
|
586
595
|
and prop.edge_source is not None
|
|
587
596
|
and prop.edge_source not in defined_views
|
|
588
597
|
):
|
|
589
|
-
errors.add(
|
|
590
|
-
ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
|
|
591
|
-
prop.edge_source, "View", view_id, "View", property_name=prop_name
|
|
592
|
-
)
|
|
593
|
-
)
|
|
598
|
+
errors.add(PropertyNotFoundError(prop.edge_source, "view", prop_name, view_id, "view"))
|
|
594
599
|
|
|
595
600
|
# This allows for multiple view properties to be mapped to the same container property,
|
|
596
601
|
# as long as they have different external_id, otherwise this will lead to raising
|
|
@@ -615,36 +620,36 @@ class DMSSchema:
|
|
|
615
620
|
== (container_id, container_property_identifier)
|
|
616
621
|
]
|
|
617
622
|
errors.add(
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
623
|
+
PropertyMappingDuplicatedError(
|
|
624
|
+
container_id,
|
|
625
|
+
"container",
|
|
626
|
+
container_property_identifier,
|
|
627
|
+
frozenset({dm.PropertyId(view_id, prop_name) for prop_name in view_properties}),
|
|
628
|
+
"view property",
|
|
622
629
|
)
|
|
623
630
|
)
|
|
624
631
|
|
|
625
632
|
if self.data_model:
|
|
626
633
|
model = self.data_model
|
|
627
634
|
if model.space not in defined_spaces:
|
|
628
|
-
errors.add(
|
|
629
|
-
ReferredResourceNotFoundError[str, dm.DataModelId](
|
|
630
|
-
model.space, "Space", model.as_id(), "Data Model"
|
|
631
|
-
)
|
|
632
|
-
)
|
|
635
|
+
errors.add(ResourceNotFoundError(model.space, "space", model.as_id(), "data model"))
|
|
633
636
|
|
|
634
637
|
view_counts: dict[dm.ViewId, int] = defaultdict(int)
|
|
635
638
|
for view_id_or_class in model.views or []:
|
|
636
639
|
view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
|
|
637
640
|
if view_id not in defined_views:
|
|
638
|
-
errors.add(
|
|
639
|
-
ReferredResourceNotFoundError[dm.ViewId, dm.DataModelId](
|
|
640
|
-
view_id, "View", model.as_id(), "DataModel"
|
|
641
|
-
)
|
|
642
|
-
)
|
|
641
|
+
errors.add(ResourceNotFoundError(view_id, "view", model.as_id(), "data model"))
|
|
643
642
|
view_counts[view_id] += 1
|
|
644
643
|
|
|
645
644
|
for view_id, count in view_counts.items():
|
|
646
645
|
if count > 1:
|
|
647
|
-
errors.add(
|
|
646
|
+
errors.add(
|
|
647
|
+
ResourceDuplicatedError(
|
|
648
|
+
view_id,
|
|
649
|
+
"view",
|
|
650
|
+
repr(model.as_id()),
|
|
651
|
+
)
|
|
652
|
+
)
|
|
648
653
|
|
|
649
654
|
return list(errors)
|
|
650
655
|
|
|
@@ -852,7 +857,9 @@ class PipelineSchema(DMSSchema):
|
|
|
852
857
|
try:
|
|
853
858
|
loaded = yaml.safe_load(yaml_file.read_text())
|
|
854
859
|
except Exception as e:
|
|
855
|
-
warnings.warn(
|
|
860
|
+
warnings.warn(
|
|
861
|
+
FileTypeUnexpectedWarning(yaml_file, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
|
|
862
|
+
)
|
|
856
863
|
continue
|
|
857
864
|
if isinstance(loaded, list):
|
|
858
865
|
data[attr_name].extend(loaded)
|
|
@@ -918,7 +925,9 @@ class PipelineSchema(DMSSchema):
|
|
|
918
925
|
try:
|
|
919
926
|
loaded = yaml.safe_load(zip_ref.read(file_info).decode())
|
|
920
927
|
except Exception as e:
|
|
921
|
-
warnings.warn(
|
|
928
|
+
warnings.warn(
|
|
929
|
+
FileTypeUnexpectedWarning(filepath, frozenset([".yaml", ".yml"]), str(e)), stacklevel=2
|
|
930
|
+
)
|
|
922
931
|
continue
|
|
923
932
|
if isinstance(loaded, list):
|
|
924
933
|
data[attr_name].extend(loaded)
|
|
@@ -3,10 +3,22 @@ from typing import Any, ClassVar
|
|
|
3
3
|
|
|
4
4
|
from cognite.client import data_modeling as dm
|
|
5
5
|
|
|
6
|
-
from cognite.neat.issues import IssueList, NeatIssueList
|
|
7
|
-
from cognite.neat.
|
|
6
|
+
from cognite.neat.issues import IssueList, NeatError, NeatIssue, NeatIssueList
|
|
7
|
+
from cognite.neat.issues.errors import (
|
|
8
|
+
PropertyDefinitionDuplicatedError,
|
|
9
|
+
ResourceChangedError,
|
|
10
|
+
ResourceNotDefinedError,
|
|
11
|
+
)
|
|
12
|
+
from cognite.neat.issues.warnings import (
|
|
13
|
+
NotSupportedHasDataFilterLimitWarning,
|
|
14
|
+
NotSupportedViewContainerLimitWarning,
|
|
15
|
+
)
|
|
16
|
+
from cognite.neat.issues.warnings.user_modeling import (
|
|
17
|
+
NotNeatSupportedFilterWarning,
|
|
18
|
+
ViewPropertyLimitWarning,
|
|
19
|
+
)
|
|
8
20
|
from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
|
|
9
|
-
from cognite.neat.rules.models._constants import
|
|
21
|
+
from cognite.neat.rules.models._constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
|
|
10
22
|
from cognite.neat.rules.models.data_types import DataType
|
|
11
23
|
from cognite.neat.rules.models.entities import ContainerEntity
|
|
12
24
|
from cognite.neat.rules.models.wrapped_entities import RawFilter
|
|
@@ -51,7 +63,7 @@ class DMSPostValidation:
|
|
|
51
63
|
if prop.container and prop.container_property:
|
|
52
64
|
container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
|
|
53
65
|
|
|
54
|
-
errors: list[
|
|
66
|
+
errors: list[NeatError] = []
|
|
55
67
|
for (container, prop_name), properties in container_properties_by_id.items():
|
|
56
68
|
if len(properties) == 1:
|
|
57
69
|
continue
|
|
@@ -63,42 +75,78 @@ class DMSPostValidation:
|
|
|
63
75
|
is_all_direct = all(prop.connection == "direct" for _, prop in properties)
|
|
64
76
|
if len(value_types) > 1 and not is_all_direct:
|
|
65
77
|
errors.append(
|
|
66
|
-
|
|
78
|
+
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
67
79
|
container_id,
|
|
80
|
+
"container",
|
|
68
81
|
prop_name,
|
|
69
|
-
|
|
70
|
-
|
|
82
|
+
frozenset({v.dms._type if isinstance(v, DataType) else str(v) for v in value_types}),
|
|
83
|
+
tuple(row_numbers),
|
|
84
|
+
"rows",
|
|
71
85
|
)
|
|
72
86
|
)
|
|
73
87
|
list_definitions = {prop.is_list for _, prop in properties if prop.is_list is not None}
|
|
74
88
|
if len(list_definitions) > 1:
|
|
75
89
|
errors.append(
|
|
76
|
-
|
|
90
|
+
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
91
|
+
container_id,
|
|
92
|
+
"container",
|
|
93
|
+
prop_name,
|
|
94
|
+
frozenset(list_definitions),
|
|
95
|
+
tuple(row_numbers),
|
|
96
|
+
"rows",
|
|
97
|
+
)
|
|
77
98
|
)
|
|
78
99
|
nullable_definitions = {prop.nullable for _, prop in properties if prop.nullable is not None}
|
|
79
100
|
if len(nullable_definitions) > 1:
|
|
80
101
|
errors.append(
|
|
81
|
-
|
|
102
|
+
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
103
|
+
container_id,
|
|
104
|
+
"container",
|
|
105
|
+
prop_name,
|
|
106
|
+
frozenset(nullable_definitions),
|
|
107
|
+
tuple(row_numbers),
|
|
108
|
+
"rows",
|
|
109
|
+
)
|
|
82
110
|
)
|
|
83
111
|
default_definitions = {prop.default for _, prop in properties if prop.default is not None}
|
|
84
112
|
if len(default_definitions) > 1:
|
|
85
113
|
errors.append(
|
|
86
|
-
|
|
87
|
-
container_id,
|
|
114
|
+
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
115
|
+
container_id,
|
|
116
|
+
"container",
|
|
117
|
+
prop_name,
|
|
118
|
+
frozenset(
|
|
119
|
+
tuple(f"{k}:{v}" for k, v in def_.items()) if isinstance(def_, dict) else def_
|
|
120
|
+
for def_ in default_definitions
|
|
121
|
+
),
|
|
122
|
+
tuple(row_numbers),
|
|
123
|
+
"rows",
|
|
88
124
|
)
|
|
89
125
|
)
|
|
90
126
|
index_definitions = {",".join(prop.index) for _, prop in properties if prop.index is not None}
|
|
91
127
|
if len(index_definitions) > 1:
|
|
92
128
|
errors.append(
|
|
93
|
-
|
|
129
|
+
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
130
|
+
container_id,
|
|
131
|
+
"container",
|
|
132
|
+
prop_name,
|
|
133
|
+
frozenset(index_definitions),
|
|
134
|
+
tuple(row_numbers),
|
|
135
|
+
"rows",
|
|
136
|
+
)
|
|
94
137
|
)
|
|
95
138
|
constraint_definitions = {
|
|
96
139
|
",".join(prop.constraint) for _, prop in properties if prop.constraint is not None
|
|
97
140
|
}
|
|
98
141
|
if len(constraint_definitions) > 1:
|
|
99
142
|
errors.append(
|
|
100
|
-
|
|
101
|
-
container_id,
|
|
143
|
+
PropertyDefinitionDuplicatedError[dm.ContainerId](
|
|
144
|
+
container_id,
|
|
145
|
+
"container",
|
|
146
|
+
prop_name,
|
|
147
|
+
frozenset(constraint_definitions),
|
|
148
|
+
tuple(row_numbers),
|
|
149
|
+
"rows",
|
|
102
150
|
)
|
|
103
151
|
)
|
|
104
152
|
|
|
@@ -125,32 +173,25 @@ class DMSPostValidation:
|
|
|
125
173
|
defined_views |= {view.view.as_id() for view in self.rules.last.views}
|
|
126
174
|
|
|
127
175
|
property_count_by_view: dict[dm.ViewId, int] = defaultdict(int)
|
|
128
|
-
errors: list[
|
|
176
|
+
errors: list[NeatIssue] = []
|
|
129
177
|
for prop_no, prop in enumerate(self.properties):
|
|
130
178
|
view_id = prop.view.as_id()
|
|
131
179
|
if view_id not in defined_views:
|
|
132
180
|
errors.append(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
)
|
|
181
|
+
ResourceNotDefinedError[dm.ViewId](
|
|
182
|
+
identifier=view_id,
|
|
183
|
+
resource_type="view",
|
|
184
|
+
location="Views Sheet",
|
|
185
|
+
column_name="View",
|
|
186
|
+
row_number=prop_no,
|
|
187
|
+
sheet_name="Properties",
|
|
188
|
+
),
|
|
142
189
|
)
|
|
143
190
|
else:
|
|
144
191
|
property_count_by_view[view_id] += 1
|
|
145
192
|
for view_id, count in property_count_by_view.items():
|
|
146
|
-
if count >
|
|
147
|
-
errors.append(
|
|
148
|
-
issues.dms.ViewSizeWarning(
|
|
149
|
-
view_id=view_id,
|
|
150
|
-
limit=DMS_CONTAINER_SIZE_LIMIT,
|
|
151
|
-
count=count,
|
|
152
|
-
)
|
|
153
|
-
)
|
|
193
|
+
if count > DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
194
|
+
errors.append(ViewPropertyLimitWarning(view_id, count))
|
|
154
195
|
if self.metadata.schema_ is SchemaCompleteness.complete:
|
|
155
196
|
defined_containers = {container.container.as_id() for container in self.containers or []}
|
|
156
197
|
if self.metadata.data_model_type == DataModelType.solution and self.rules.reference:
|
|
@@ -161,28 +202,26 @@ class DMSPostValidation:
|
|
|
161
202
|
for prop_no, prop in enumerate(self.properties):
|
|
162
203
|
if prop.container and (container_id := prop.container.as_id()) not in defined_containers:
|
|
163
204
|
errors.append(
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
url=None,
|
|
205
|
+
ResourceNotDefinedError[dm.ContainerId](
|
|
206
|
+
identifier=container_id,
|
|
207
|
+
resource_type="container",
|
|
208
|
+
location="Containers Sheet",
|
|
209
|
+
column_name="Container",
|
|
210
|
+
row_number=prop_no,
|
|
211
|
+
sheet_name="Properties",
|
|
172
212
|
)
|
|
173
213
|
)
|
|
174
214
|
for _container_no, container in enumerate(self.containers or []):
|
|
175
215
|
for constraint_no, constraint in enumerate(container.constraint or []):
|
|
176
216
|
if constraint.as_id() not in defined_containers:
|
|
177
217
|
errors.append(
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
url=None,
|
|
218
|
+
ResourceNotDefinedError[dm.ContainerId](
|
|
219
|
+
identifier=constraint.as_id(),
|
|
220
|
+
resource_type="container",
|
|
221
|
+
location="Containers Sheet",
|
|
222
|
+
column_name="Constraint",
|
|
223
|
+
row_number=constraint_no,
|
|
224
|
+
sheet_name="Properties",
|
|
186
225
|
)
|
|
187
226
|
)
|
|
188
227
|
self.issue_list.extend(errors)
|
|
@@ -212,10 +251,11 @@ class DMSPostValidation:
|
|
|
212
251
|
new_dumped, existing_dumped
|
|
213
252
|
)
|
|
214
253
|
self.issue_list.append(
|
|
215
|
-
|
|
216
|
-
container_id
|
|
217
|
-
|
|
218
|
-
|
|
254
|
+
ResourceChangedError(
|
|
255
|
+
container_id,
|
|
256
|
+
"container",
|
|
257
|
+
changed_properties=frozenset(changed_properties),
|
|
258
|
+
changed_attributes=frozenset(changed_attributes),
|
|
219
259
|
)
|
|
220
260
|
)
|
|
221
261
|
|
|
@@ -241,10 +281,11 @@ class DMSPostValidation:
|
|
|
241
281
|
# Only added new properties, no problem
|
|
242
282
|
continue
|
|
243
283
|
self.issue_list.append(
|
|
244
|
-
|
|
245
|
-
view_id
|
|
246
|
-
|
|
247
|
-
|
|
284
|
+
ResourceChangedError(
|
|
285
|
+
view_id,
|
|
286
|
+
"view",
|
|
287
|
+
changed_properties=frozenset(changed_properties),
|
|
288
|
+
changed_attributes=frozenset(changed_attributes),
|
|
248
289
|
)
|
|
249
290
|
)
|
|
250
291
|
|
|
@@ -254,9 +295,9 @@ class DMSPostValidation:
|
|
|
254
295
|
|
|
255
296
|
if mapped_containers and len(mapped_containers) > 10:
|
|
256
297
|
self.issue_list.append(
|
|
257
|
-
|
|
258
|
-
view_id
|
|
259
|
-
|
|
298
|
+
NotSupportedViewContainerLimitWarning(
|
|
299
|
+
view_id,
|
|
300
|
+
len(mapped_containers),
|
|
260
301
|
)
|
|
261
302
|
)
|
|
262
303
|
if (
|
|
@@ -265,9 +306,9 @@ class DMSPostValidation:
|
|
|
265
306
|
and len(view.filter.dump()["hasData"]) > 10
|
|
266
307
|
):
|
|
267
308
|
self.issue_list.append(
|
|
268
|
-
|
|
269
|
-
view_id
|
|
270
|
-
|
|
309
|
+
NotSupportedHasDataFilterLimitWarning(
|
|
310
|
+
view_id,
|
|
311
|
+
len(view.filter.dump()["hasData"]),
|
|
271
312
|
)
|
|
272
313
|
)
|
|
273
314
|
|
|
@@ -275,9 +316,7 @@ class DMSPostValidation:
|
|
|
275
316
|
for view in self.views:
|
|
276
317
|
if view.filter_ and isinstance(view.filter_, RawFilter):
|
|
277
318
|
self.issue_list.append(
|
|
278
|
-
|
|
279
|
-
view_id=view.view.as_id(),
|
|
280
|
-
)
|
|
319
|
+
NotNeatSupportedFilterWarning(view.view.as_id()),
|
|
281
320
|
)
|
|
282
321
|
|
|
283
322
|
@staticmethod
|
|
@@ -12,7 +12,7 @@ from cognite.neat.rules.models._base import (
|
|
|
12
12
|
SchemaCompleteness,
|
|
13
13
|
SheetList,
|
|
14
14
|
)
|
|
15
|
-
from cognite.neat.rules.models._constants import
|
|
15
|
+
from cognite.neat.rules.models._constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
|
|
16
16
|
from cognite.neat.rules.models.data_types import DataType
|
|
17
17
|
from cognite.neat.rules.models.domain import DomainRules
|
|
18
18
|
from cognite.neat.rules.models.entities import (
|
|
@@ -266,7 +266,7 @@ class _InformationRulesConverter:
|
|
|
266
266
|
else:
|
|
267
267
|
container_entity = prop.class_.as_container_entity(default_space)
|
|
268
268
|
|
|
269
|
-
while self.property_count_by_container[container_entity] >=
|
|
269
|
+
while self.property_count_by_container[container_entity] >= DMS_CONTAINER_PROPERTY_SIZE_LIMIT:
|
|
270
270
|
container_entity.suffix = self._bump_suffix(container_entity.suffix)
|
|
271
271
|
|
|
272
272
|
self.property_count_by_container[container_entity] += 1
|
|
@@ -8,9 +8,7 @@ from pydantic.main import IncEx
|
|
|
8
8
|
from rdflib import Namespace
|
|
9
9
|
|
|
10
10
|
from cognite.neat.constants import get_default_prefixes
|
|
11
|
-
from cognite.neat.issues import
|
|
12
|
-
from cognite.neat.rules import issues
|
|
13
|
-
from cognite.neat.rules.issues.spreadsheet import DefaultValueTypeNotProperError
|
|
11
|
+
from cognite.neat.issues.errors import PropertyDefinitionError
|
|
14
12
|
from cognite.neat.rules.models._base import (
|
|
15
13
|
BaseMetadata,
|
|
16
14
|
BaseRules,
|
|
@@ -228,11 +226,12 @@ class InformationProperty(SheetEntity):
|
|
|
228
226
|
self.default = self.value_type.python(self.default)
|
|
229
227
|
|
|
230
228
|
except Exception:
|
|
231
|
-
raise
|
|
229
|
+
raise PropertyDefinitionError(
|
|
230
|
+
self.class_,
|
|
231
|
+
"Class",
|
|
232
232
|
self.property_,
|
|
233
|
-
type
|
|
234
|
-
|
|
235
|
-
).as_exception() from None
|
|
233
|
+
f"Default value {self.default} is not of type {self.value_type.python}",
|
|
234
|
+
) from None
|
|
236
235
|
return self
|
|
237
236
|
|
|
238
237
|
@property
|
|
@@ -306,7 +305,7 @@ class InformationRules(BaseRules):
|
|
|
306
305
|
if issue_list.warnings:
|
|
307
306
|
issue_list.trigger_warnings()
|
|
308
307
|
if issue_list.has_errors:
|
|
309
|
-
raise
|
|
308
|
+
raise issue_list.as_exception()
|
|
310
309
|
return self
|
|
311
310
|
|
|
312
311
|
def dump(
|