cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/workflows/steps/lib/current/rules_importer.py

@@ -5,11 +5,13 @@ from typing import ClassVar
 from cognite.client import CogniteClient
 from cognite.client.data_classes.data_modeling import DataModelId
 
+from cognite.neat.issues.errors import WorkflowStepNotInitializedError
 from cognite.neat.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules import importers
+from cognite.neat.rules._shared import InputRules
 from cognite.neat.rules.models import RoleTypes
 from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
-from cognite.neat.workflows._exceptions import StepNotInitialized
+from cognite.neat.rules.transformers import ImporterPipeline
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import MultiRuleData
 from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -55,7 +57,7 @@ class ExcelToRules(Step):
 
     def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         file_name = self.configs.get("File name", None)
         full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
@@ -70,17 +72,17 @@ class ExcelToRules(Step):
 
         # if role is None, it will be inferred from the rules file
         role = self.configs.get("Role")
-        role_enum = None
+        role_enum: RoleTypes | None = None
         if role != "infer" and role is not None:
             role_enum = RoleTypes[role]
 
-        excel_importer = importers.ExcelImporter(rules_file_path)
-        rules, issues = excel_importer.to_rules(errors="continue", role=role_enum)
+        excel_importer = importers.ExcelImporter[InputRules](rules_file_path)
+        result = ImporterPipeline.try_verify(excel_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -91,7 +93,7 @@ class ExcelToRules(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class OntologyToRules(Step):
@@ -124,7 +126,7 @@ class OntologyToRules(Step):
 
     def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         file_name = self.configs.get("File name", None)
         full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
@@ -144,12 +146,12 @@ class OntologyToRules(Step):
             role_enum = RoleTypes[role]
 
         ontology_importer = importers.OWLImporter(filepath=rules_file_path)
-        rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(ontology_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -160,7 +162,7 @@ class OntologyToRules(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class IMFToRules(Step):
@@ -193,7 +195,7 @@ class IMFToRules(Step):
 
     def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         file_name = self.configs.get("File name", None)
 
@@ -215,12 +217,12 @@ class IMFToRules(Step):
             role_enum = RoleTypes[role]
 
         ontology_importer = importers.IMFImporter(filepath=rules_file_path)
-        rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(ontology_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -231,7 +233,7 @@ class IMFToRules(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class DMSToRules(Step):
@@ -271,7 +273,7 @@ class DMSToRules(Step):
 
     def run(self, cdf_client: CogniteClient) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         datamodel_id_str = self.configs.get("Data model id")
         if datamodel_id_str is None:
@@ -305,12 +307,12 @@ class DMSToRules(Step):
         if role != "infer" and role is not None:
             role_enum = RoleTypes[role]
 
-        rules, issues = dms_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(dms_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -321,7 +323,7 @@ class DMSToRules(Step):
 
         output_text = "Rules import and validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)
 
 
 class RulesInferenceFromRdfFile(Step):
@@ -360,7 +362,7 @@ class RulesInferenceFromRdfFile(Step):
 
     def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         file_path = self.configs.get("File path", None)
         full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
@@ -388,12 +390,12 @@ class RulesInferenceFromRdfFile(Step):
         inference_importer = importers.InferenceImporter.from_rdf_file(
             rdf_file_path, max_number_of_instance=max_number_of_instance
         )
-        rules, issues = inference_importer.to_rules(errors="continue", role=role_enum)
+        result = ImporterPipeline.try_verify(inference_importer, role_enum)
 
-        if rules is None:
+        if result.rules is None:
             output_dir = self.config.staging_path
             report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
-            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_writer.write_to_file(result.issues, file_or_dir_path=output_dir)
             report_file = report_writer.default_file_name
             error_text = (
                 "<p></p>"
@@ -404,4 +406,4 @@ class RulesInferenceFromRdfFile(Step):
 
         output_text = "Rules validation passed successfully!"
 
-        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(result.rules)

cognite/neat/workflows/steps/lib/current/rules_validator.py

@@ -6,11 +6,10 @@ from typing import ClassVar
 from cognite.client import CogniteClient
 
 from cognite.neat.issues import NeatIssueList
-from cognite.neat.issues.errors.resources import ReferredResourceNotFoundError
+from cognite.neat.issues.errors import ResourceNotFoundError, WorkflowStepNotInitializedError
 from cognite.neat.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules.models import DMSRules, SchemaCompleteness
 from cognite.neat.utils.cdf.loaders import ViewLoader
-from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import MultiRuleData
 from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -42,7 +41,7 @@ class ValidateRulesAgainstCDF(Step):
 
     def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage:  # type: ignore[override, syntax]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         if not isinstance(rules.dms, DMSRules):
             return FlowMessage(
@@ -64,17 +63,17 @@ class ValidateRulesAgainstCDF(Step):
         missing_spaces = [
             error.identifier
             for error in errors
-            if isinstance(error, ReferredResourceNotFoundError) and error.resource_type == "Space"
+            if isinstance(error, ResourceNotFoundError) and error.resource_type == "Space"
         ]
         missing_views = [
             error.identifier
             for error in errors
-            if isinstance(error, ReferredResourceNotFoundError) and error.resource_type == "View"
+            if isinstance(error, ResourceNotFoundError) and error.resource_type == "View"
         ]
         missing_containers = [
             error.identifier
             for error in errors
-            if isinstance(error, ReferredResourceNotFoundError) and error.resource_type == "Container"
+            if isinstance(error, ResourceNotFoundError) and error.resource_type == "Container"
         ]
 
         retrieved_spaces = cdf_client.data_modeling.spaces.retrieve(missing_spaces).as_write()

cognite/neat/workflows/steps/lib/io/io_steps.py

@@ -6,7 +6,7 @@ from typing import ClassVar
 import requests
 from cognite.client import CogniteClient
 
-from cognite.neat.workflows._exceptions import StepNotInitialized
+from cognite.neat.issues.errors import WorkflowStepNotInitializedError
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.step_model import Configurable, Step
 
@@ -51,7 +51,7 @@ class DownloadFileFromGitHub(Step):
 
     def run(self) -> FlowMessage:  # type: ignore[override, syntax]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
         github_filepath = self.configs["github.filepath"]
         github_personal_token = self.configs["github.personal_token"]
         github_owner = self.configs["github.owner"]
@@ -136,7 +136,7 @@ class UploadFileToGitHub(Step):
 
     def run(self) -> FlowMessage:  # type: ignore[override, syntax]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
         github_filepath = self.configs["github.filepath"]
         github_personal_token = self.configs["github.personal_token"]
         github_owner = self.configs["github.owner"]
@@ -210,7 +210,7 @@ class DownloadFileFromCDF(Step):
 
     def run(self, cdf_client: CogniteClient) -> FlowMessage:  # type: ignore[override, syntax]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
 
         output_dir = self.data_store_path / Path(self.configs["local.storage_dir"])
         output_dir.mkdir(parents=True, exist_ok=True)
@@ -244,7 +244,7 @@ class UploadFileToCDF(Step):
 
     def run(self, cdf_client: CogniteClient) -> FlowMessage:  # type: ignore[override, syntax]
         if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
+            raise WorkflowStepNotInitializedError(type(self).__name__)
         full_local_file_path = (
             self.data_store_path / Path(self.configs["local.storage_dir"]) / self.configs["local.file_name"]
         )

cognite/neat/workflows/steps_registry.py

@@ -14,8 +14,7 @@ import cognite.neat.workflows.steps.lib.current
 import cognite.neat.workflows.steps.lib.io
 from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
 from cognite.neat.config import Config
-from cognite.neat.exceptions import InvalidWorkFlowError
-from cognite.neat.workflows._exceptions import ConfigurationNotSet
+from cognite.neat.issues.errors import WorkflowConfigurationNotSetError, WorkFlowMissingDataError
 from cognite.neat.workflows.model import FlowMessage, WorkflowConfigs
 from cognite.neat.workflows.steps.step_model import Configurable, DataContract, Step
 
@@ -71,7 +70,7 @@ class StepsRegistry:
 
     def load_workflow_step_classes(self, workflow_name: str):
         if not self.data_store_path:
-            raise ConfigurationNotSet("data_store_path")
+            raise WorkflowConfigurationNotSetError("data_store_path")
         workflow_steps_path = Path(self.data_store_path) / "workflows" / workflow_name
         if workflow_steps_path.exists():
             self.load_custom_step_classes(workflow_steps_path, scope="workflow")
@@ -164,9 +163,9 @@ class StepsRegistry:
                         missing_data.append(parameter.annotation.__name__)
                         continue
                 if not is_valid:
-                    raise InvalidWorkFlowError(step_name, missing_data)
+                    raise WorkFlowMissingDataError(step_name, frozenset(missing_data))
                 return step_obj.run(*input_data)
-        raise InvalidWorkFlowError(step_name, [])
+        raise WorkFlowMissingDataError(step_name, frozenset({}))
 
     def get_list_of_steps(self) -> list[StepMetadata]:
         steps: list[StepMetadata] = []

{cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.88.2
+Version: 0.89.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0