cognite-neat 0.88.3__py3-none-any.whl → 0.90.0__py3-none-any.whl

cognite/neat/utils/cdf/data_classes.py

@@ -268,6 +268,10 @@ class ViewApplyDict(CogniteResourceDict[ViewId, ViewApply]):
     def _as_id(cls, resource: ViewApply) -> ViewId:
         return resource.as_id()
 
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[ViewApply]) -> "ViewApplyDict":
+        return cls({view.as_id(): view for view in iterable})
+
 
 class SpaceApplyDict(CogniteResourceDict[str, SpaceApply]):
     _RESOURCE = SpaceApply
@@ -276,6 +280,10 @@ class SpaceApplyDict(CogniteResourceDict[str, SpaceApply]):
     def _as_id(cls, resource: SpaceApply) -> str:
         return resource.space
 
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[SpaceApply]) -> "SpaceApplyDict":
+        return cls({space.space: space for space in iterable})
+
 
 class ContainerApplyDict(CogniteResourceDict[ContainerId, ContainerApply]):
     _RESOURCE = ContainerApply
@@ -284,6 +292,10 @@ class ContainerApplyDict(CogniteResourceDict[ContainerId, ContainerApply]):
     def _as_id(cls, resource: ContainerApply) -> ContainerId:
         return resource.as_id()
 
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[ContainerApply]) -> "ContainerApplyDict":
+        return cls({container.as_id(): container for container in iterable})
+
 
 class DataModelApplyDict(CogniteResourceDict[DataModelId, DataModelApply]):
     _RESOURCE = DataModelApply
@@ -292,6 +304,10 @@ class DataModelApplyDict(CogniteResourceDict[DataModelId, DataModelApply]):
     def _as_id(cls, resource: DataModelApply) -> DataModelId:
         return resource.as_id()
 
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[DataModelApply]) -> "DataModelApplyDict":
+        return cls({data_model.as_id(): data_model for data_model in iterable})
+
 
 class NodeApplyDict(CogniteResourceDict[NodeId, NodeApply]):
     _RESOURCE = NodeApply
@@ -299,3 +315,7 @@ class NodeApplyDict(CogniteResourceDict[NodeId, NodeApply]):
     @classmethod
     def _as_id(cls, resource: NodeApply) -> NodeId:
         return resource.as_id()
+
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[NodeApply]) -> "NodeApplyDict":
+        return cls({node.as_id(): node for node in iterable})

cognite/neat/utils/regex_patterns.py

@@ -19,6 +19,12 @@ PROPERTY_ID_COMPLIANCE_REGEX = r"^(\*)|(?!^(Property|property)$)(^[a-zA-Z][a-zA-
 VERSION_COMPLIANCE_REGEX = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
 
 
+# This pattern ignores commas inside brackets
+SPLIT_ON_COMMA_PATTERN = re.compile(r",(?![^(]*\))")
+# This pattern ignores equal signs inside brackets
+SPLIT_ON_EQUAL_PATTERN = re.compile(r"=(?![^(]*\))")
+
+
 class _Patterns:
     @cached_property
     def more_than_one_alphanumeric(self) -> re.Pattern:

cognite/neat/workflows/steps/lib/current/rules_exporter.py

@@ -5,7 +5,14 @@ from typing import ClassVar, Literal, cast
 from cognite.neat.issues.errors import WorkflowStepNotInitializedError
 from cognite.neat.rules import exporters
 from cognite.neat.rules._shared import DMSRules, InformationRules, VerifiedRules
-from cognite.neat.rules.models import RoleTypes
+from cognite.neat.rules.models import AssetRules, RoleTypes
+from cognite.neat.rules.transformers import (
+    AssetToInformation,
+    DMSToInformation,
+    InformationToAsset,
+    InformationToDMS,
+    RulesPipeline,
+)
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import CogniteClient, MultiRuleData
 from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -75,26 +82,30 @@ class DeleteDataModelFromCDF(Step):
                 error_text="No DMS Schema components selected for removal! Please select minimum one!",
                 step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
             )
-        input_rules = rules.dms or rules.information
+        input_rules = rules.dms or rules.information or rules.asset
         if input_rules is None:
             return FlowMessage(
                 error_text="Missing DMS or Information rules in the input data! "
                 "Please ensure that a DMS or Information rules is provided!",
                 step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
             )
+        if isinstance(input_rules, DMSRules):
+            dms_rules = input_rules
+        elif isinstance(input_rules, InformationRules):
+            dms_rules = InformationToDMS().transform(input_rules).rules
+        elif isinstance(input_rules, AssetRules):
+            dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
+        else:
+            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
 
         dms_exporter = exporters.DMSExporter(
             export_components=frozenset(components_to_delete),
-            include_space=(
-                None
-                if multi_space_components_delete
-                else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
-            ),
+            include_space=(None if multi_space_components_delete else {dms_rules.metadata.space}),
         )
 
         report_lines = ["# Data Model Deletion from CDF\n\n"]
         errors = []
-        for result in dms_exporter.delete_from_cdf(rules=input_rules, client=cdf_client, dry_run=dry_run):
+        for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=cdf_client, dry_run=dry_run):
             report_lines.append(str(result))
             errors.extend(result.error_messages)
 
@@ -192,14 +203,18 @@ class RulesToDMS(Step):
                 "Please ensure that a DMS or Information rules is provided!",
                 step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
             )
+        if isinstance(input_rules, DMSRules):
+            dms_rules = input_rules
+        elif isinstance(input_rules, InformationRules):
+            dms_rules = InformationToDMS().transform(input_rules).rules
+        elif isinstance(input_rules, AssetRules):
+            dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
+        else:
+            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
 
         dms_exporter = exporters.DMSExporter(
             export_components=frozenset(components_to_create),
-            include_space=(
-                None
-                if multi_space_components_create
-                else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
-            ),
+            include_space=(None if multi_space_components_create else {dms_rules.metadata.space}),
             existing_handling=existing_components_handling,
         )
 
@@ -212,11 +227,11 @@ class RulesToDMS(Step):
         )
         schema_zip = f"{file_name}.zip"
         schema_full_path = output_dir / schema_zip
-        dms_exporter.export_to_file(input_rules, schema_full_path)
+        dms_exporter.export_to_file(dms_rules, schema_full_path)
 
         report_lines = ["# DMS Schema Export to CDF\n\n"]
         errors = []
-        for result in dms_exporter.export_to_cdf_iterable(rules=input_rules, client=cdf_client, dry_run=dry_run):
+        for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
             report_lines.append(str(result))
             errors.extend(result.error_messages)
 
@@ -296,7 +311,7 @@ class RulesToExcel(Step):
         dump_format = self.configs.get("Dump Format", "user")
         styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
         role = self.configs.get("Output role format")
-        output_role = None
+        output_role: RoleTypes | None = None
         if role != "input" and role is not None:
             output_role = RoleTypes[role]
 
@@ -310,13 +325,9 @@ class RulesToExcel(Step):
                 step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
             )
 
-        excel_exporter = exporters.ExcelExporter(
-            styling=styling,
-            output_role=output_role,
-            dump_as=dump_format,  # type: ignore[arg-type]
-            new_model_id=new_model_id,
-        )
+        excel_exporter = exporters.ExcelExporter(styling=styling, dump_as=dump_format, new_model_id=new_model_id)  # type: ignore[arg-type]
 
+        # Todo - Move the conversion to a separate workflow step.
         rule_instance: VerifiedRules
         if rules.domain:
             rule_instance = rules.domain
@@ -324,9 +335,42 @@ class RulesToExcel(Step):
             rule_instance = rules.information
         elif rules.dms:
             rule_instance = rules.dms
+        elif rules.asset:
+            rule_instance = rules.asset
         else:
             output_errors = "No rules provided for export!"
             return FlowMessage(error_text=output_errors, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
+
+        if rule_instance.metadata.role is output_role or output_role is None:
+            ...
+        elif output_role is RoleTypes.dms:
+            if isinstance(rule_instance, InformationRules):
+                rule_instance = InformationToDMS().transform(rule_instance).rules
+            elif isinstance(rule_instance, AssetRules):
+                rule_instance = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(
+                    rule_instance
+                )
+            else:
+                raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
+        elif output_role is RoleTypes.information:
+            if isinstance(rule_instance, DMSRules):
+                rule_instance = DMSToInformation().transform(rule_instance).rules
+            elif isinstance(rule_instance, AssetRules):
+                rule_instance = AssetToInformation().transform(rule_instance).rules
+            else:
+                raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
+        elif output_role is RoleTypes.asset:
+            if isinstance(rule_instance, InformationRules):
+                rule_instance = InformationToAsset().transform(rule_instance).rules
+            elif isinstance(rule_instance, DMSRules):
+                rule_instance = RulesPipeline[DMSRules, AssetRules]([DMSToInformation(), InformationToAsset()]).run(
+                    rule_instance
+                )
+            else:
+                raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
+        else:
+            raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
+
         if output_role is None:
             output_role = rule_instance.metadata.role
         output_dir = self.data_store_path / Path("staging")
@@ -384,8 +428,16 @@ class RulesToOntology(Step):
         )
         storage_path.parent.mkdir(parents=True, exist_ok=True)
 
+        input_rules = rules.information or rules.dms
+        if isinstance(input_rules, DMSRules):
+            info_rules = DMSToInformation().transform(input_rules).rules
+        elif isinstance(input_rules, InformationRules):
+            info_rules = input_rules
+        else:
+            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
+
         exporter = exporters.OWLExporter()
-        exporter.export_to_file(cast(InformationRules | DMSRules, rules.information or rules.dms), storage_path)
+        exporter.export_to_file(info_rules, storage_path)
 
         relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
 
@@ -435,8 +487,16 @@ class RulesToSHACL(Step):
         )
         storage_path.parent.mkdir(parents=True, exist_ok=True)
 
+        input_rules = rules.information or rules.dms
+        if isinstance(input_rules, DMSRules):
+            info_rules = DMSToInformation().transform(input_rules).rules
+        elif isinstance(input_rules, InformationRules):
+            info_rules = input_rules
+        else:
+            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
+
         exporter = exporters.SHACLExporter()
-        exporter.export_to_file(cast(InformationRules | DMSRules, rules.information or rules.dms), storage_path)
+        exporter.export_to_file(info_rules, storage_path)
 
         relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
 
@@ -485,9 +545,15 @@ class RulesToSemanticDataModel(Step):
             self.data_store_path / Path(self.configs["File path"]) if self.configs["File path"] else default_path
         )
         storage_path.parent.mkdir(parents=True, exist_ok=True)
-
+        input_rules = rules.information or rules.dms
+        if isinstance(input_rules, DMSRules):
+            info_rules = DMSToInformation().transform(input_rules).rules
+        elif isinstance(input_rules, InformationRules):
+            info_rules = input_rules
+        else:
+            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
         exporter = exporters.SemanticDataModelExporter()
-        exporter.export_to_file(cast(InformationRules | DMSRules, rules.information or rules.dms), storage_path)
+        exporter.export_to_file(info_rules, storage_path)
 
         relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
 
@@ -534,28 +600,31 @@ class RulesToCDFTransformations(Step):
                 "Please ensure that a DMS or Information rules is provided!",
                 step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
             )
-        default_instance_space = (
-            input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix
-        )
-        instance_space = self.configs.get("Instance space") or default_instance_space
+        if isinstance(input_rules, DMSRules):
+            dms_rules = input_rules
+        elif isinstance(input_rules, InformationRules):
+            dms_rules = InformationToDMS().transform(input_rules).rules
+        elif isinstance(input_rules, AssetRules):
+            dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
+        else:
+            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
+
+        instance_space = self.configs.get("Instance space") or dms_rules.metadata.space
         dry_run = self.configs.get("Dry run", "False") == "True"
         dms_exporter = exporters.DMSExporter(
             export_pipeline=True, instance_space=instance_space, export_components=["spaces"]
         )
         output_dir = self.config.staging_path
         output_dir.mkdir(parents=True, exist_ok=True)
-        file_name = (
-            input_rules.metadata.external_id
-            if isinstance(input_rules, DMSRules)
-            else input_rules.metadata.name.replace(" ", "_").lower()
-        )
+        file_name = dms_rules.metadata.external_id.replace(":", "_")
         schema_zip = f"{file_name}_pipeline.zip"
         schema_full_path = output_dir / schema_zip
-        dms_exporter.export_to_file(input_rules, schema_full_path)
+
+        dms_exporter.export_to_file(dms_rules, schema_full_path)
 
         report_lines = ["# DMS Schema Export to CDF\n\n"]
         errors = []
-        for result in dms_exporter.export_to_cdf_iterable(rules=input_rules, client=cdf_client, dry_run=dry_run):
+        for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
             report_lines.append(str(result))
             errors.extend(result.error_messages)
 

cognite/neat/workflows/steps/lib/current/rules_importer.py

@@ -8,8 +8,10 @@ from cognite.client.data_classes.data_modeling import DataModelId
 from cognite.neat.issues.errors import WorkflowStepNotInitializedError
 from cognite.neat.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules import importers
+from cognite.neat.rules._shared import InputRules
 from cognite.neat.rules.models import RoleTypes
 from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
+from cognite.neat.rules.transformers import ImporterPipeline
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import MultiRuleData
 from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -70,17 +72,17 @@ class ExcelToRules(Step):
 
         # if role is None, it will be inferred from the rules file
         role = self.configs.get("Role")
-        role_enum = None
+        role_enum: RoleTypes | None = None
         if role != "infer" and role is not None:
             role_enum = RoleTypes[role]
 
-        excel_importer = importers.ExcelImporter(rules_file_path)
-        rules, issues = excel_importer.to_rules(errors="continue", role=role_enum)
+        excel_importer = importers.ExcelImporter[InputRules](rules_file_path)
+        result = ImporterPipeline.try_verify(excel_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -91,7 +93,7 @@ class ExcelToRules(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class OntologyToRules(Step):
@@ -144,12 +146,12 @@ class OntologyToRules(Step):
             role_enum = RoleTypes[role]
 
         ontology_importer = importers.OWLImporter(filepath=rules_file_path)
-        rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(ontology_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -160,7 +162,7 @@ class OntologyToRules(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class IMFToRules(Step):
@@ -215,12 +217,12 @@ class IMFToRules(Step):
             role_enum = RoleTypes[role]
 
         ontology_importer = importers.IMFImporter(filepath=rules_file_path)
-        rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(ontology_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -231,7 +233,7 @@ class IMFToRules(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class DMSToRules(Step):
@@ -305,12 +307,12 @@ class DMSToRules(Step):
         if role != "infer" and role is not None:
             role_enum = RoleTypes[role]
 
-        rules, issues = dms_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(dms_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -321,7 +323,7 @@ class DMSToRules(Step):
 
         output_text = "Rules import and validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class RulesInferenceFromRdfFile(Step):
@@ -388,12 +390,12 @@ class RulesInferenceFromRdfFile(Step):
         inference_importer = importers.InferenceImporter.from_rdf_file(
             rdf_file_path, max_number_of_instance=max_number_of_instance
         )
-        rules, issues = inference_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(inference_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -404,4 +406,4 @@ class RulesInferenceFromRdfFile(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)

{cognite_neat-0.88.3.dist-info → cognite_neat-0.90.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.88.3
+Version: 0.90.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0