cognite-neat 0.72.2__py3-none-any.whl → 0.73.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +3 -2
- cognite/neat/graph/extractor/_graph_capturing_sheet.py +14 -12
- cognite/neat/rules/examples/power-grid-containers.yaml +3 -0
- cognite/neat/rules/examples/power-grid-model.yaml +3 -0
- cognite/neat/rules/exporter/_rules2dms.py +3 -10
- cognite/neat/rules/exporter/_rules2excel.py +3 -2
- cognite/neat/rules/exporters/_rules2dms.py +3 -7
- cognite/neat/rules/exporters/_rules2excel.py +94 -4
- cognite/neat/rules/exporters/_rules2ontology.py +2 -0
- cognite/neat/rules/importer/_dms2rules.py +1 -4
- cognite/neat/rules/importers/_dms2rules.py +40 -11
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +15 -15
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +2 -0
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +1 -1
- cognite/neat/rules/importers/_spreadsheet2rules.py +52 -31
- cognite/neat/rules/issues/base.py +9 -1
- cognite/neat/rules/issues/dms.py +74 -0
- cognite/neat/rules/models/_rules/_types/_base.py +6 -16
- cognite/neat/rules/models/_rules/base.py +24 -2
- cognite/neat/rules/models/_rules/dms_architect_rules.py +181 -71
- cognite/neat/rules/models/_rules/dms_schema.py +5 -1
- cognite/neat/rules/models/_rules/domain_rules.py +14 -1
- cognite/neat/rules/models/_rules/information_rules.py +16 -2
- cognite/neat/utils/spreadsheet.py +2 -2
- cognite/neat/workflows/steps/lib/rules_exporter.py +0 -1
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/METADATA +2 -2
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/RECORD +31 -32
- cognite/neat/py.typed +0 -0
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.72.2.dist-info → cognite_neat-0.73.0.dist-info}/entry_points.txt +0 -0
|
@@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
|
|
|
10
10
|
from cognite.client import data_modeling as dm
|
|
11
11
|
from cognite.client.data_classes.data_modeling import PropertyType as CognitePropertyType
|
|
12
12
|
from cognite.client.data_classes.data_modeling.containers import BTreeIndex
|
|
13
|
-
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
14
13
|
from cognite.client.data_classes.data_modeling.views import SingleReverseDirectRelationApply, ViewPropertyApply
|
|
15
14
|
from pydantic import Field, field_validator, model_serializer, model_validator
|
|
16
15
|
from pydantic_core.core_schema import SerializationInfo, ValidationInfo
|
|
@@ -19,7 +18,6 @@ from rdflib import Namespace
|
|
|
19
18
|
import cognite.neat.rules.issues.spreadsheet
|
|
20
19
|
from cognite.neat.rules import issues
|
|
21
20
|
from cognite.neat.rules.models._rules.domain_rules import DomainRules
|
|
22
|
-
from cognite.neat.utils.text import to_camel
|
|
23
21
|
|
|
24
22
|
from ._types import (
|
|
25
23
|
CdfValueType,
|
|
@@ -135,18 +133,21 @@ class DMSMetadata(BaseMetadata):
|
|
|
135
133
|
)
|
|
136
134
|
|
|
137
135
|
@classmethod
|
|
138
|
-
def
|
|
139
|
-
|
|
140
|
-
if data_model.description and (description_match := re.search(r"Creator: (.+)", data_model.description)):
|
|
136
|
+
def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
|
|
137
|
+
if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):
|
|
141
138
|
creator = description_match.group(1).split(", ")
|
|
142
|
-
|
|
143
|
-
elif
|
|
139
|
+
description = description_raw.replace(description_match.string, "").strip() or None
|
|
140
|
+
elif description_raw:
|
|
144
141
|
creator = ["MISSING"]
|
|
145
|
-
description =
|
|
142
|
+
description = description_raw
|
|
146
143
|
else:
|
|
147
144
|
creator = ["MISSING"]
|
|
148
|
-
description =
|
|
145
|
+
description = None
|
|
146
|
+
return description, creator
|
|
149
147
|
|
|
148
|
+
@classmethod
|
|
149
|
+
def from_data_model(cls, data_model: dm.DataModelApply) -> "DMSMetadata":
|
|
150
|
+
description, creator = cls._get_description_and_creator(data_model.description)
|
|
150
151
|
return cls(
|
|
151
152
|
schema_=SchemaCompleteness.complete,
|
|
152
153
|
space=data_model.space,
|
|
@@ -207,11 +208,11 @@ class DMSContainer(SheetEntity):
|
|
|
207
208
|
reference: ReferenceType = Field(alias="Reference", default=None)
|
|
208
209
|
constraint: ContainerListType | None = Field(None, alias="Constraint")
|
|
209
210
|
|
|
210
|
-
def as_container(self, default_space: str
|
|
211
|
-
container_id = self.container.as_id(default_space
|
|
211
|
+
def as_container(self, default_space: str) -> dm.ContainerApply:
|
|
212
|
+
container_id = self.container.as_id(default_space)
|
|
212
213
|
constraints: dict[str, dm.Constraint] = {}
|
|
213
214
|
for constraint in self.constraint or []:
|
|
214
|
-
requires = dm.RequiresConstraint(constraint.as_id(default_space
|
|
215
|
+
requires = dm.RequiresConstraint(constraint.as_id(default_space))
|
|
215
216
|
constraints[f"{constraint.space}_{constraint.external_id}"] = requires
|
|
216
217
|
|
|
217
218
|
return dm.ContainerApply(
|
|
@@ -246,18 +247,15 @@ class DMSView(SheetEntity):
|
|
|
246
247
|
filter_: Literal["hasData", "nodeType"] | None = Field(None, alias="Filter")
|
|
247
248
|
in_model: bool = Field(True, alias="InModel")
|
|
248
249
|
|
|
249
|
-
def as_view(self, default_space: str, default_version: str
|
|
250
|
-
view_id = self.view.as_id(False, default_space, default_version
|
|
250
|
+
def as_view(self, default_space: str, default_version: str) -> dm.ViewApply:
|
|
251
|
+
view_id = self.view.as_id(False, default_space, default_version)
|
|
251
252
|
return dm.ViewApply(
|
|
252
253
|
space=view_id.space,
|
|
253
254
|
external_id=view_id.external_id,
|
|
254
255
|
version=view_id.version or default_version,
|
|
255
256
|
name=self.name or None,
|
|
256
257
|
description=self.description,
|
|
257
|
-
implements=[
|
|
258
|
-
parent.as_id(False, default_space, default_version, standardize_casing)
|
|
259
|
-
for parent in self.implements or []
|
|
260
|
-
]
|
|
258
|
+
implements=[parent.as_id(False, default_space, default_version) for parent in self.implements or []]
|
|
261
259
|
or None,
|
|
262
260
|
properties={},
|
|
263
261
|
)
|
|
@@ -494,12 +492,117 @@ class DMSRules(BaseRules):
|
|
|
494
492
|
raise issues.MultiValueError(errors)
|
|
495
493
|
return self
|
|
496
494
|
|
|
495
|
+
@model_validator(mode="after")
|
|
496
|
+
def validate_extension(self) -> "DMSRules":
|
|
497
|
+
if self.metadata.schema_ is not SchemaCompleteness.extended:
|
|
498
|
+
return self
|
|
499
|
+
if not self.reference:
|
|
500
|
+
raise ValueError("The schema is set to 'extended', but no reference rules are provided to validate against")
|
|
501
|
+
is_solution = self.metadata.space != self.reference.metadata.space
|
|
502
|
+
if is_solution:
|
|
503
|
+
return self
|
|
504
|
+
if self.metadata.extension is ExtensionCategory.rebuild:
|
|
505
|
+
# Everything is allowed
|
|
506
|
+
return self
|
|
507
|
+
# Is an extension of an existing model.
|
|
508
|
+
user_schema = self.as_schema()
|
|
509
|
+
ref_schema = self.reference.as_schema()
|
|
510
|
+
new_containers = {container.as_id(): container for container in user_schema.containers}
|
|
511
|
+
existing_containers = {container.as_id(): container for container in ref_schema.containers}
|
|
512
|
+
|
|
513
|
+
errors: list[issues.NeatValidationError] = []
|
|
514
|
+
for container_id, container in new_containers.items():
|
|
515
|
+
existing_container = existing_containers.get(container_id)
|
|
516
|
+
if not existing_container or existing_container == container:
|
|
517
|
+
# No problem
|
|
518
|
+
continue
|
|
519
|
+
new_dumped = container.dump()
|
|
520
|
+
existing_dumped = existing_container.dump()
|
|
521
|
+
changed_attributes, changed_properties = self._changed_attributes_and_properties(
|
|
522
|
+
new_dumped, existing_dumped
|
|
523
|
+
)
|
|
524
|
+
errors.append(
|
|
525
|
+
issues.dms.ChangingContainerError(
|
|
526
|
+
container_id=container_id,
|
|
527
|
+
changed_properties=changed_properties or None,
|
|
528
|
+
changed_attributes=changed_attributes or None,
|
|
529
|
+
)
|
|
530
|
+
)
|
|
531
|
+
|
|
532
|
+
if self.metadata.extension is ExtensionCategory.reshape and errors:
|
|
533
|
+
raise issues.MultiValueError(errors)
|
|
534
|
+
elif self.metadata.extension is ExtensionCategory.reshape:
|
|
535
|
+
# Reshape allows changes to views
|
|
536
|
+
return self
|
|
537
|
+
|
|
538
|
+
new_views = {view.as_id(): view for view in user_schema.views}
|
|
539
|
+
existing_views = {view.as_id(): view for view in ref_schema.views}
|
|
540
|
+
for view_id, view in new_views.items():
|
|
541
|
+
existing_view = existing_views.get(view_id)
|
|
542
|
+
if not existing_view or existing_view == view:
|
|
543
|
+
# No problem
|
|
544
|
+
continue
|
|
545
|
+
changed_attributes, changed_properties = self._changed_attributes_and_properties(
|
|
546
|
+
view.dump(), existing_view.dump()
|
|
547
|
+
)
|
|
548
|
+
errors.append(
|
|
549
|
+
issues.dms.ChangingViewError(
|
|
550
|
+
view_id=view_id,
|
|
551
|
+
changed_properties=changed_properties or None,
|
|
552
|
+
changed_attributes=changed_attributes or None,
|
|
553
|
+
)
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
if errors:
|
|
557
|
+
raise issues.MultiValueError(errors)
|
|
558
|
+
return self
|
|
559
|
+
|
|
560
|
+
@staticmethod
|
|
561
|
+
def _changed_attributes_and_properties(
|
|
562
|
+
new_dumped: dict[str, Any], existing_dumped: dict[str, Any]
|
|
563
|
+
) -> tuple[list[str], list[str]]:
|
|
564
|
+
"""Helper method to find the changed attributes and properties between two containers or views."""
|
|
565
|
+
new_attributes = {key: value for key, value in new_dumped.items() if key != "properties"}
|
|
566
|
+
existing_attributes = {key: value for key, value in existing_dumped.items() if key != "properties"}
|
|
567
|
+
changed_attributes = [key for key in new_attributes if new_attributes[key] != existing_attributes.get(key)]
|
|
568
|
+
new_properties = new_dumped.get("properties", {})
|
|
569
|
+
existing_properties = existing_dumped.get("properties", {})
|
|
570
|
+
changed_properties = [prop for prop in new_properties if new_properties[prop] != existing_properties.get(prop)]
|
|
571
|
+
return changed_attributes, changed_properties
|
|
572
|
+
|
|
497
573
|
@model_validator(mode="after")
|
|
498
574
|
def validate_schema(self) -> "DMSRules":
|
|
499
|
-
if self.metadata.schema_ is
|
|
575
|
+
if self.metadata.schema_ is SchemaCompleteness.partial:
|
|
500
576
|
return self
|
|
577
|
+
elif self.metadata.schema_ is SchemaCompleteness.complete:
|
|
578
|
+
rules: DMSRules = self
|
|
579
|
+
elif self.metadata.schema_ is SchemaCompleteness.extended:
|
|
580
|
+
if not self.reference:
|
|
581
|
+
raise ValueError(
|
|
582
|
+
"The schema is set to 'extended', but no reference rules are provided to validate against"
|
|
583
|
+
)
|
|
584
|
+
# This is an extension of the reference rules, we need to merge the two
|
|
585
|
+
rules = self.copy(deep=True)
|
|
586
|
+
rules.properties.extend(self.reference.properties.data)
|
|
587
|
+
existing_views = {view.view.as_id(False) for view in rules.views}
|
|
588
|
+
rules.views.extend([view for view in self.reference.views if view.view.as_id(False) not in existing_views])
|
|
589
|
+
if rules.containers and self.reference.containers:
|
|
590
|
+
existing_containers = {
|
|
591
|
+
container.container.as_id(self.metadata.space) for container in rules.containers.data
|
|
592
|
+
}
|
|
593
|
+
rules.containers.extend(
|
|
594
|
+
[
|
|
595
|
+
container
|
|
596
|
+
for container in self.reference.containers
|
|
597
|
+
if container.container.as_id(self.reference.metadata.space) not in existing_containers
|
|
598
|
+
]
|
|
599
|
+
)
|
|
600
|
+
elif not rules.containers and self.reference.containers:
|
|
601
|
+
rules.containers = self.reference.containers
|
|
602
|
+
else:
|
|
603
|
+
raise ValueError("Unknown schema completeness")
|
|
501
604
|
|
|
502
|
-
schema =
|
|
605
|
+
schema = rules.as_schema()
|
|
503
606
|
errors = schema.validate()
|
|
504
607
|
if errors:
|
|
505
608
|
raise issues.MultiValueError(errors)
|
|
@@ -557,17 +660,19 @@ class DMSRules(BaseRules):
|
|
|
557
660
|
)
|
|
558
661
|
containers.append(dumped)
|
|
559
662
|
|
|
560
|
-
|
|
663
|
+
output = {
|
|
561
664
|
"Metadata" if info.by_alias else "metadata": self.metadata.model_dump(**kwargs),
|
|
562
665
|
"Properties" if info.by_alias else "properties": properties,
|
|
563
666
|
"Views" if info.by_alias else "views": views,
|
|
564
667
|
"Containers" if info.by_alias else "containers": containers,
|
|
668
|
+
"is_reference": self.is_reference,
|
|
565
669
|
}
|
|
670
|
+
if self.reference is not None:
|
|
671
|
+
output["Reference" if info.by_alias else "reference"] = self.reference.model_dump(**kwargs)
|
|
672
|
+
return output
|
|
566
673
|
|
|
567
|
-
def as_schema(
|
|
568
|
-
|
|
569
|
-
) -> DMSSchema:
|
|
570
|
-
return _DMSExporter(standardize_casing, include_pipeline, instance_space).to_schema(self)
|
|
674
|
+
def as_schema(self, include_pipeline: bool = False, instance_space: str | None = None) -> DMSSchema:
|
|
675
|
+
return _DMSExporter(include_pipeline, instance_space).to_schema(self)
|
|
571
676
|
|
|
572
677
|
def as_information_architect_rules(self) -> "InformationRules":
|
|
573
678
|
return _DMSRulesConverter(self).as_information_architect_rules()
|
|
@@ -599,17 +704,12 @@ class _DMSExporter:
|
|
|
599
704
|
(This module cannot have a dependency on the exporter module, as it would create a circular dependency.)
|
|
600
705
|
|
|
601
706
|
Args
|
|
602
|
-
standardize_casing (bool): If True, the casing of the identifiers will be standardized. This means external IDs
|
|
603
|
-
are PascalCase and property names are camelCase.
|
|
604
707
|
include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
|
|
605
708
|
raw tables and transformations necessary to populate the data model.
|
|
606
709
|
instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
|
|
607
710
|
"""
|
|
608
711
|
|
|
609
|
-
def __init__(
|
|
610
|
-
self, standardize_casing: bool = True, include_pipeline: bool = False, instance_space: str | None = None
|
|
611
|
-
):
|
|
612
|
-
self.standardize_casing = standardize_casing
|
|
712
|
+
def __init__(self, include_pipeline: bool = False, instance_space: str | None = None):
|
|
613
713
|
self.include_pipeline = include_pipeline
|
|
614
714
|
self.instance_space = instance_space
|
|
615
715
|
|
|
@@ -628,9 +728,7 @@ class _DMSExporter:
|
|
|
628
728
|
)
|
|
629
729
|
|
|
630
730
|
views_not_in_model = {
|
|
631
|
-
view.view.as_id(False, default_space, default_version
|
|
632
|
-
for view in rules.views
|
|
633
|
-
if not view.in_model
|
|
731
|
+
view.view.as_id(False, default_space, default_version) for view in rules.views if not view.in_model
|
|
634
732
|
}
|
|
635
733
|
data_model = rules.metadata.as_data_model()
|
|
636
734
|
data_model.views = sorted(
|
|
@@ -675,12 +773,9 @@ class _DMSExporter:
|
|
|
675
773
|
default_space: str,
|
|
676
774
|
default_version: str,
|
|
677
775
|
) -> tuple[dm.ViewApplyList, dm.NodeApplyList]:
|
|
678
|
-
views = dm.ViewApplyList(
|
|
679
|
-
[dms_view.as_view(default_space, default_version, self.standardize_casing) for dms_view in dms_views]
|
|
680
|
-
)
|
|
776
|
+
views = dm.ViewApplyList([dms_view.as_view(default_space, default_version) for dms_view in dms_views])
|
|
681
777
|
dms_view_by_id = {
|
|
682
|
-
dms_view.view.as_id(False, default_space, default_version
|
|
683
|
-
for dms_view in dms_views
|
|
778
|
+
dms_view.view.as_id(False, default_space, default_version): dms_view for dms_view in dms_views
|
|
684
779
|
}
|
|
685
780
|
|
|
686
781
|
for view in views:
|
|
@@ -694,7 +789,7 @@ class _DMSExporter:
|
|
|
694
789
|
# This is not yet supported in the CDF API, a warning has already been issued, here we convert it to
|
|
695
790
|
# a multi-edge connection.
|
|
696
791
|
if isinstance(prop.value_type, ViewEntity):
|
|
697
|
-
source = prop.value_type.as_id(False, default_space, default_version
|
|
792
|
+
source = prop.value_type.as_id(False, default_space, default_version)
|
|
698
793
|
else:
|
|
699
794
|
raise ValueError(
|
|
700
795
|
"Direct relation must have a view as value type. "
|
|
@@ -709,43 +804,48 @@ class _DMSExporter:
|
|
|
709
804
|
direction="outwards",
|
|
710
805
|
)
|
|
711
806
|
elif prop.container and prop.container_property and prop.view_property:
|
|
712
|
-
container_prop_identifier =
|
|
713
|
-
to_camel(prop.container_property) if self.standardize_casing else prop.container_property
|
|
714
|
-
)
|
|
807
|
+
container_prop_identifier = prop.container_property
|
|
715
808
|
extra_args: dict[str, Any] = {}
|
|
716
809
|
if prop.relation == "direct" and isinstance(prop.value_type, ViewEntity):
|
|
717
|
-
extra_args["source"] = prop.value_type.as_id(
|
|
718
|
-
True, default_space, default_version, self.standardize_casing
|
|
719
|
-
)
|
|
810
|
+
extra_args["source"] = prop.value_type.as_id(True, default_space, default_version)
|
|
720
811
|
elif prop.relation == "direct" and not isinstance(prop.value_type, ViewEntity):
|
|
721
812
|
raise ValueError(
|
|
722
813
|
"Direct relation must have a view as value type. "
|
|
723
814
|
"This should have been validated in the rules"
|
|
724
815
|
)
|
|
725
816
|
view_property = dm.MappedPropertyApply(
|
|
726
|
-
container=prop.container.as_id(default_space
|
|
817
|
+
container=prop.container.as_id(default_space),
|
|
727
818
|
container_property_identifier=container_prop_identifier,
|
|
728
819
|
**extra_args,
|
|
729
820
|
)
|
|
730
821
|
elif prop.view and prop.view_property and prop.relation == "multiedge":
|
|
731
822
|
if isinstance(prop.value_type, ViewEntity):
|
|
732
|
-
source = prop.value_type.as_id(False, default_space, default_version
|
|
823
|
+
source = prop.value_type.as_id(False, default_space, default_version)
|
|
733
824
|
else:
|
|
734
825
|
raise ValueError(
|
|
735
826
|
"Multiedge relation must have a view as value type. "
|
|
736
827
|
"This should have been validated in the rules"
|
|
737
828
|
)
|
|
738
|
-
|
|
739
|
-
|
|
829
|
+
if isinstance(prop.reference, ReferenceEntity):
|
|
830
|
+
ref_view_prop = prop.reference.as_prop_id(default_space, default_version)
|
|
831
|
+
edge_type = dm.DirectRelationReference(
|
|
832
|
+
space=ref_view_prop.source.space,
|
|
833
|
+
external_id=f"{ref_view_prop.source.external_id}.{ref_view_prop.property}",
|
|
834
|
+
)
|
|
835
|
+
else:
|
|
836
|
+
edge_type = dm.DirectRelationReference(
|
|
740
837
|
space=source.space,
|
|
741
838
|
external_id=f"{prop.view.external_id}.{prop.view_property}",
|
|
742
|
-
)
|
|
839
|
+
)
|
|
840
|
+
|
|
841
|
+
view_property = dm.MultiEdgeConnectionApply(
|
|
842
|
+
type=edge_type,
|
|
743
843
|
source=source,
|
|
744
844
|
direction="outwards",
|
|
745
845
|
)
|
|
746
846
|
elif prop.view and prop.view_property and prop.relation == "reversedirect":
|
|
747
847
|
if isinstance(prop.value_type, ViewPropEntity):
|
|
748
|
-
source = prop.value_type.as_id(False, default_space, default_version
|
|
848
|
+
source = prop.value_type.as_id(False, default_space, default_version)
|
|
749
849
|
else:
|
|
750
850
|
raise ValueError(
|
|
751
851
|
"Reverse direct relation must have a view as value type. "
|
|
@@ -765,11 +865,19 @@ class _DMSExporter:
|
|
|
765
865
|
warnings.warn(
|
|
766
866
|
issues.dms.ReverseOfDirectRelationListWarning(view_id, prop.property_), stacklevel=2
|
|
767
867
|
)
|
|
768
|
-
|
|
769
|
-
|
|
868
|
+
if isinstance(reverse_prop.reference, ReferenceEntity):
|
|
869
|
+
ref_view_prop = reverse_prop.reference.as_prop_id(default_space, default_version)
|
|
870
|
+
edge_type = dm.DirectRelationReference(
|
|
871
|
+
space=ref_view_prop.source.space,
|
|
872
|
+
external_id=f"{ref_view_prop.source.external_id}.{ref_view_prop.property}",
|
|
873
|
+
)
|
|
874
|
+
else:
|
|
875
|
+
edge_type = dm.DirectRelationReference(
|
|
770
876
|
space=source.space,
|
|
771
877
|
external_id=f"{reverse_prop.view.external_id}.{reverse_prop.view_property}",
|
|
772
|
-
)
|
|
878
|
+
)
|
|
879
|
+
view_property = dm.MultiEdgeConnectionApply(
|
|
880
|
+
type=edge_type,
|
|
773
881
|
source=source,
|
|
774
882
|
direction="inwards",
|
|
775
883
|
)
|
|
@@ -797,16 +905,24 @@ class _DMSExporter:
|
|
|
797
905
|
continue
|
|
798
906
|
else:
|
|
799
907
|
continue
|
|
800
|
-
prop_name =
|
|
908
|
+
prop_name = prop.view_property
|
|
801
909
|
view.properties[prop_name] = view_property
|
|
802
910
|
|
|
803
911
|
node_types = dm.NodeApplyList([])
|
|
804
912
|
parent_views = {parent for view in views for parent in view.implements or []}
|
|
805
913
|
for view in views:
|
|
806
914
|
ref_containers = sorted(view.referenced_containers(), key=lambda c: c.as_tuple())
|
|
807
|
-
has_data = dm.filters.HasData(containers=list(ref_containers)) if ref_containers else None
|
|
808
|
-
node_type = dm.filters.Equals(["node", "type"], {"space": view.space, "externalId": view.external_id})
|
|
809
915
|
dms_view = dms_view_by_id.get(view.as_id())
|
|
916
|
+
has_data = dm.filters.HasData(containers=list(ref_containers)) if ref_containers else None
|
|
917
|
+
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
|
|
918
|
+
# If the view is a reference, we implement the reference view,
|
|
919
|
+
# and need the filter to match the reference
|
|
920
|
+
ref_view = dms_view.reference.as_id(False, default_space, default_version)
|
|
921
|
+
node_type = dm.filters.Equals(
|
|
922
|
+
["node", "type"], {"space": ref_view.space, "externalId": ref_view.external_id}
|
|
923
|
+
)
|
|
924
|
+
else:
|
|
925
|
+
node_type = dm.filters.Equals(["node", "type"], {"space": view.space, "externalId": view.external_id})
|
|
810
926
|
if view.as_id() in parent_views:
|
|
811
927
|
if dms_view and dms_view.filter_ == "nodeType":
|
|
812
928
|
warnings.warn(issues.dms.NodeTypeFilterOnParentViewWarning(view.as_id()), stacklevel=2)
|
|
@@ -838,10 +954,7 @@ class _DMSExporter:
|
|
|
838
954
|
default_space: str,
|
|
839
955
|
) -> dm.ContainerApplyList:
|
|
840
956
|
containers = dm.ContainerApplyList(
|
|
841
|
-
[
|
|
842
|
-
dms_container.as_container(default_space, self.standardize_casing)
|
|
843
|
-
for dms_container in dms_container or []
|
|
844
|
-
]
|
|
957
|
+
[dms_container.as_container(default_space) for dms_container in dms_container or []]
|
|
845
958
|
)
|
|
846
959
|
container_to_drop = set()
|
|
847
960
|
for container in containers:
|
|
@@ -858,7 +971,7 @@ class _DMSExporter:
|
|
|
858
971
|
else:
|
|
859
972
|
type_cls = dm.DirectRelation
|
|
860
973
|
|
|
861
|
-
prop_name =
|
|
974
|
+
prop_name = prop.container_property
|
|
862
975
|
|
|
863
976
|
if type_cls is dm.DirectRelation:
|
|
864
977
|
container.properties[prop_name] = dm.ContainerProperty(
|
|
@@ -870,10 +983,7 @@ class _DMSExporter:
|
|
|
870
983
|
)
|
|
871
984
|
else:
|
|
872
985
|
type_: CognitePropertyType
|
|
873
|
-
|
|
874
|
-
type_ = type_cls(is_list=prop.is_list or False)
|
|
875
|
-
else:
|
|
876
|
-
type_ = cast(CognitePropertyType, type_cls())
|
|
986
|
+
type_ = type_cls(is_list=prop.is_list or False)
|
|
877
987
|
container.properties[prop_name] = dm.ContainerProperty(
|
|
878
988
|
type=type_,
|
|
879
989
|
nullable=prop.nullable if prop.nullable is not None else True,
|
|
@@ -919,21 +1029,21 @@ class _DMSExporter:
|
|
|
919
1029
|
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]] = defaultdict(list)
|
|
920
1030
|
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
|
|
921
1031
|
for prop in rules.properties:
|
|
922
|
-
view_id = prop.view.as_id(False, default_space, default_version
|
|
1032
|
+
view_id = prop.view.as_id(False, default_space, default_version)
|
|
923
1033
|
view_properties_by_id[view_id].append(prop)
|
|
924
1034
|
|
|
925
1035
|
if prop.container and prop.container_property:
|
|
926
1036
|
if prop.relation == "direct" and prop.is_list:
|
|
927
1037
|
warnings.warn(
|
|
928
1038
|
issues.dms.DirectRelationListWarning(
|
|
929
|
-
container_id=prop.container.as_id(default_space
|
|
930
|
-
view_id=prop.view.as_id(False, default_space, default_version
|
|
1039
|
+
container_id=prop.container.as_id(default_space),
|
|
1040
|
+
view_id=prop.view.as_id(False, default_space, default_version),
|
|
931
1041
|
property=prop.container_property,
|
|
932
1042
|
),
|
|
933
1043
|
stacklevel=2,
|
|
934
1044
|
)
|
|
935
1045
|
continue
|
|
936
|
-
container_id = prop.container.as_id(default_space
|
|
1046
|
+
container_id = prop.container.as_id(default_space)
|
|
937
1047
|
container_properties_by_id[container_id].append(prop)
|
|
938
1048
|
|
|
939
1049
|
return container_properties_by_id, view_properties_by_id
|
|
@@ -63,6 +63,10 @@ class DMSSchema:
|
|
|
63
63
|
if len(data_models) == 0:
|
|
64
64
|
raise ValueError(f"Data model {data_model_id} not found")
|
|
65
65
|
data_model = data_models.latest_version()
|
|
66
|
+
return cls.from_data_model(client, data_model)
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
def from_data_model(cls, client: CogniteClient, data_model: dm.DataModel) -> "DMSSchema":
|
|
66
70
|
views = dm.ViewList(data_model.views)
|
|
67
71
|
container_ids = views.referenced_containers()
|
|
68
72
|
containers = client.data_modeling.containers.retrieve(list(container_ids))
|
|
@@ -267,7 +271,7 @@ class DMSSchema:
|
|
|
267
271
|
if isinstance(item, dm.ContainerApply | dm.ViewApply | dm.DataModelApply | dm.NodeApply | RawTableWrite):
|
|
268
272
|
identifier = item.as_id().as_tuple()
|
|
269
273
|
if len(identifier) == 3 and identifier[2] is None:
|
|
270
|
-
return identifier[:2]
|
|
274
|
+
return identifier[:2] # type: ignore[misc]
|
|
271
275
|
return cast(tuple[str, str] | tuple[str, str, str], identifier)
|
|
272
276
|
elif isinstance(item, dm.SpaceApply):
|
|
273
277
|
return item.space
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import math
|
|
1
2
|
from typing import Any, ClassVar
|
|
2
3
|
|
|
3
|
-
from pydantic import Field, model_serializer
|
|
4
|
+
from pydantic import Field, field_serializer, field_validator, model_serializer
|
|
4
5
|
from pydantic_core.core_schema import SerializationInfo
|
|
5
6
|
|
|
6
7
|
from ._types import ParentClassType, PropertyType, SemanticValueType, StrOrListType
|
|
@@ -24,6 +25,18 @@ class DomainProperty(SheetEntity):
|
|
|
24
25
|
min_count: int | None = Field(alias="Min Count", default=None)
|
|
25
26
|
max_count: int | float | None = Field(alias="Max Count", default=None)
|
|
26
27
|
|
|
28
|
+
@field_serializer("max_count", when_used="json-unless-none")
|
|
29
|
+
def serialize_max_count(self, value: int | float | None) -> int | float | None | str:
|
|
30
|
+
if isinstance(value, float) and math.isinf(value):
|
|
31
|
+
return None
|
|
32
|
+
return value
|
|
33
|
+
|
|
34
|
+
@field_validator("max_count", mode="before")
|
|
35
|
+
def parse_max_count(cls, value: int | float | None) -> int | float | None:
|
|
36
|
+
if value is None:
|
|
37
|
+
return float("inf")
|
|
38
|
+
return value
|
|
39
|
+
|
|
27
40
|
|
|
28
41
|
class DomainClass(SheetEntity):
|
|
29
42
|
description: str | None = Field(None, alias="Description")
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import math
|
|
1
2
|
import re
|
|
2
3
|
import sys
|
|
3
4
|
import warnings
|
|
@@ -5,7 +6,7 @@ from collections import defaultdict
|
|
|
5
6
|
from datetime import datetime
|
|
6
7
|
from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
|
|
7
8
|
|
|
8
|
-
from pydantic import Field, field_validator, model_serializer, model_validator
|
|
9
|
+
from pydantic import Field, field_serializer, field_validator, model_serializer, model_validator
|
|
9
10
|
from pydantic_core.core_schema import SerializationInfo
|
|
10
11
|
from rdflib import Namespace
|
|
11
12
|
|
|
@@ -47,6 +48,7 @@ from ._types._base import Unknown
|
|
|
47
48
|
from .base import (
|
|
48
49
|
BaseMetadata,
|
|
49
50
|
ExtensionCategory,
|
|
51
|
+
ExtensionCategoryType,
|
|
50
52
|
MatchType,
|
|
51
53
|
RoleTypes,
|
|
52
54
|
RuleModel,
|
|
@@ -69,7 +71,7 @@ else:
|
|
|
69
71
|
class InformationMetadata(BaseMetadata):
|
|
70
72
|
role: ClassVar[RoleTypes] = RoleTypes.information_architect
|
|
71
73
|
schema_: SchemaCompleteness = Field(alias="schema")
|
|
72
|
-
extension:
|
|
74
|
+
extension: ExtensionCategoryType | None = ExtensionCategory.addition
|
|
73
75
|
prefix: PrefixType
|
|
74
76
|
namespace: NamespaceType
|
|
75
77
|
|
|
@@ -156,6 +158,18 @@ class InformationProperty(SheetEntity):
|
|
|
156
158
|
)
|
|
157
159
|
comment: str | None = Field(alias="Comment", default=None)
|
|
158
160
|
|
|
161
|
+
@field_serializer("max_count", when_used="json-unless-none")
|
|
162
|
+
def serialize_max_count(self, value: int | float | None) -> int | float | None | str:
|
|
163
|
+
if isinstance(value, float) and math.isinf(value):
|
|
164
|
+
return None
|
|
165
|
+
return value
|
|
166
|
+
|
|
167
|
+
@field_validator("max_count", mode="before")
|
|
168
|
+
def parse_max_count(cls, value: int | float | None) -> int | float | None:
|
|
169
|
+
if value is None:
|
|
170
|
+
return float("inf")
|
|
171
|
+
return value
|
|
172
|
+
|
|
159
173
|
@model_validator(mode="after")
|
|
160
174
|
def is_valid_rule(self):
|
|
161
175
|
# TODO: Can we skip rule_type and simply try to parse the rule and if it fails, raise an error?
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from dataclasses import dataclass, field
|
|
2
|
-
from typing import Literal, overload
|
|
2
|
+
from typing import Literal, cast, overload
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
from openpyxl import load_workbook
|
|
@@ -58,7 +58,7 @@ def read_individual_sheet(
|
|
|
58
58
|
expected_headers: list[str] | None = None,
|
|
59
59
|
) -> tuple[list[dict], SpreadsheetRead] | list[dict]:
|
|
60
60
|
if expected_headers:
|
|
61
|
-
target_row = _get_row_number(load_workbook(excel_file)[sheet_name], expected_headers)
|
|
61
|
+
target_row = _get_row_number(cast(Worksheet, load_workbook(excel_file)[sheet_name]), expected_headers)
|
|
62
62
|
skiprows = target_row - 1 if target_row is not None else 0
|
|
63
63
|
else:
|
|
64
64
|
skiprows = 0
|
|
@@ -102,7 +102,6 @@ class RulesToDMS(Step):
|
|
|
102
102
|
if multi_space_components_create
|
|
103
103
|
else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix},
|
|
104
104
|
existing_handling=existing_components_handling,
|
|
105
|
-
standardize_casing=False,
|
|
106
105
|
)
|
|
107
106
|
|
|
108
107
|
output_dir = self.data_store_path / Path("staging")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cognite-neat
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.73.0
|
|
4
4
|
Summary: Knowledge graph transformation
|
|
5
5
|
Home-page: https://cognite-neat.readthedocs-hosted.com/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -19,7 +19,7 @@ Provides-Extra: graphql
|
|
|
19
19
|
Provides-Extra: oxi
|
|
20
20
|
Requires-Dist: PyYAML
|
|
21
21
|
Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
|
|
22
|
-
Requires-Dist: cognite-sdk (>=7.
|
|
22
|
+
Requires-Dist: cognite-sdk (>=7.37.0,<8.0.0)
|
|
23
23
|
Requires-Dist: deepdiff
|
|
24
24
|
Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
|
|
25
25
|
Requires-Dist: fastapi (>=0.100,<0.101)
|