cognite-neat 0.88.1__py3-none-any.whl → 0.88.2__py3-none-any.whl

cognite/neat/rules/issues/dms.py

@@ -10,13 +10,6 @@ __all__ = [
     "DMSSchemaError",
     "DMSSchemaWarning",
     "IncompleteSchemaError",
-    "MissingSpaceError",
-    "MissingContainerError",
-    "MissingContainerPropertyError",
-    "MissingViewError",
-    "MissingParentViewError",
-    "MissingSourceViewError",
-    "MissingEdgeViewError",
     "DirectRelationMissingSourceWarning",
     "ViewModelVersionNotMatchingWarning",
     "ViewModelSpaceNotMatchingWarning",
@@ -87,133 +80,6 @@ class IncompleteSchemaError(DMSSchemaError):
         return output
 
 
-@dataclass(frozen=True)
-class MissingSpaceError(DMSSchemaError):
-    description = "The spaced referred to by the Container/View/Node/Edge/DataModel does not exist"
-    fix = "Create the space"
-    space: str
-    referred_by: dm.ContainerId | dm.ViewId | dm.NodeId | dm.EdgeId | dm.DataModelId
-
-    def message(self) -> str:
-        return f"The space {self.space} referred to by {self.referred_by} does not exist"
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["space"] = self.space
-        output["referred_by"] = self.referred_by
-        return output
-
-
-@dataclass(frozen=True)
-class MissingContainerError(DMSSchemaError):
-    description = "The container referred to by the View does not exist"
-    fix = "Create the container"
-    error_name: ClassVar[str] = "MissingContainer"
-    container: dm.ContainerId
-    referred_by: dm.ViewId | dm.ContainerId
-
-    def message(self) -> str:
-        return f"The container {self.container} referred to by {self.referred_by} does not exist"
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["container"] = self.container
-        output["referred_by"] = self.referred_by
-        return output
-
-
-@dataclass(frozen=True)
-class MissingContainerPropertyError(DMSSchemaError):
-    description = "The property referred to by the View does not exist in the container"
-    fix = "Create the property"
-    error_name: ClassVar[str] = "MissingContainerProperty"
-    container: dm.ContainerId
-    property: str
-    referred_by: dm.ViewId
-
-    def message(self) -> str:
-        return (
-            f"The property {self.property} referred to by the container {self.container} "
-            f"does not exist in {self.referred_by}"
-        )
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["container"] = self.container
-        output["property"] = self.property
-        output["referred_by"] = self.referred_by
-        return output
-
-
-@dataclass(frozen=True)
-class MissingViewError(DMSSchemaError):
-    description = "The view referred to by the View/DataModel does not exist"
-    fix = "Create the view"
-    error_name: ClassVar[str] = "MissingView"
-    view: dm.ViewId
-    referred_by: dm.DataModelId | dm.ViewId
-
-    def message(self) -> str:
-        return f"The view {self.view} referred to by {self.referred_by} does not exist"
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["view"] = self.view
-        output["referred_by"] = self.referred_by
-        return output
-
-
-@dataclass(frozen=True)
-class MissingParentViewError(MissingViewError):
-    description = "The parent view referred to by the View does not exist"
-    fix = "Create the parent view"
-    error_name: ClassVar[str] = "MissingParentView"
-    referred_by: dm.ViewId
-
-    def message(self) -> str:
-        return f"The parent view {self.view} referred to by {self.referred_by} does not exist"
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["referred_by"] = self.referred_by
-        return output
-
-
-@dataclass(frozen=True)
-class MissingSourceViewError(MissingViewError):
-    description = "The source view referred to by the View does not exist"
-    fix = "Create the source view"
-    error_name: ClassVar[str] = "MissingSourceView"
-    property: str
-    referred_by: dm.ViewId
-
-    def message(self) -> str:
-        return f"The source view {self.view} referred to by {self.referred_by}.{self.property} does not exist"
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["property"] = self.property
-        return output
-
-
-@dataclass(frozen=True)
-class MissingEdgeViewError(MissingViewError):
-    description = "The edge view referred to by the View does not exist"
-    fix = "Create the edge view"
-    error_name: ClassVar[str] = "MissingEdgeView"
-    property: str
-    referred_by: dm.ViewId
-
-    def message(self) -> str:
-        return f"The edge view {self.view} referred to by {self.referred_by}.{self.property} does not exist"
-
-    def dump(self) -> dict[str, Any]:
-        output = super().dump()
-        output["property"] = self.property
-        output["referred_by"] = self.referred_by
-        return output
-
-
 @dataclass(frozen=True)
 class DuplicatedViewInDataModelError(DMSSchemaError):
     description = "The view is duplicated in the DataModel"

cognite/neat/rules/issues/spreadsheet.py

@@ -9,7 +9,7 @@ from cognite.client.data_classes.data_modeling import ContainerId, ViewId
 from pydantic_core import ErrorDetails
 from rdflib import Namespace
 
-from cognite.neat.issues import MultiValueError
+from cognite.neat.issues import MultiValueError, NeatError
 from cognite.neat.utils.spreadsheet import SpreadsheetRead
 
 from .base import DefaultPydanticError, NeatValidationError
@@ -59,8 +59,8 @@ class InvalidSheetError(NeatValidationError, ABC):
         errors: list[ErrorDetails],
         read_info_by_sheet: dict[str, SpreadsheetRead] | None = None,
         **kwargs: Any,
-    ) -> "list[NeatValidationError]":
-        output: list[NeatValidationError] = []
+    ) -> "list[NeatError]":
+        output: list[NeatError] = []
         for error in errors:
             if raised_error := error.get("ctx", {}).get("error"):
                 if isinstance(raised_error, MultiValueError):

cognite/neat/rules/models/_types/_field.py

@@ -16,7 +16,7 @@ from pydantic import (
 from pydantic.functional_serializers import PlainSerializer
 from pydantic_core import PydanticCustomError
 
-from cognite.neat.rules.issues.importing import MoreThanOneNonAlphanumericCharacterWarning
+from cognite.neat.issues.neat_warnings.identifier import RegexViolationWarning
 from cognite.neat.rules.issues.spreadsheet import RegexViolationError
 from cognite.neat.utils.regex_patterns import (
     PATTERNS,
@@ -91,7 +91,10 @@ def _property_validation(value: str) -> str:
     if not PATTERNS.property_id_compliance.match(value):
         _raise(RegexViolationError(value, PROPERTY_ID_COMPLIANCE_REGEX).as_pydantic_exception())
     if PATTERNS.more_than_one_alphanumeric.search(value):
-        warnings.warn(MoreThanOneNonAlphanumericCharacterWarning("property", value), stacklevel=2)
+        warnings.warn(
+            RegexViolationWarning(value, PROPERTY_ID_COMPLIANCE_REGEX, "property", "MoreThanOneNonAlphanumeric"),
+            stacklevel=2,
+        )
     return value
 
 

cognite/neat/rules/models/asset/_validation.py

@@ -1,8 +1,8 @@
 from graphlib import CycleError
 from typing import cast
 
+from cognite.neat.issues import IssueList
 from cognite.neat.rules import issues
-from cognite.neat.rules.issues.base import IssueList
 from cognite.neat.rules.models._base import SheetList
 from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
 from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity

cognite/neat/rules/models/dms/_schema.py

@@ -25,20 +25,16 @@ from cognite.client.data_classes.data_modeling.views import (
 )
 from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
 
+from cognite.neat.issues import NeatError
+from cognite.neat.issues.errors.properties import ReferredPropertyNotFoundError
+from cognite.neat.issues.errors.resources import ReferredResourceNotFoundError
+from cognite.neat.issues.neat_warnings.resources import FailedLoadingResourcesWarning, MultipleResourcesWarning
 from cognite.neat.rules import issues
 from cognite.neat.rules.issues.dms import (
     ContainerPropertyUsedMultipleTimesError,
     DirectRelationMissingSourceWarning,
-    DMSSchemaError,
     DuplicatedViewInDataModelError,
     IncompleteSchemaError,
-    MissingContainerError,
-    MissingContainerPropertyError,
-    MissingEdgeViewError,
-    MissingParentViewError,
-    MissingSourceViewError,
-    MissingSpaceError,
-    MissingViewError,
     MissingViewInModelWarning,
 )
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
@@ -168,14 +164,10 @@ class DMSSchema:
             connection_referenced_view_ids |= cls._connection_references(view)
         connection_referenced_view_ids = connection_referenced_view_ids - existing_view_ids
         if connection_referenced_view_ids:
-            warnings.warn(
-                MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), UserWarning, stacklevel=2
-            )
+            warnings.warn(MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), stacklevel=2)
             connection_referenced_views = view_loader.retrieve(list(connection_referenced_view_ids))
             if failed := connection_referenced_view_ids - set(connection_referenced_views.as_ids()):
-                warnings.warn(
-                    issues.importing.FailedImportWarning({repr(v) for v in failed}), UserWarning, stacklevel=2
-                )
+                warnings.warn(FailedLoadingResourcesWarning[dm.ViewId](frozenset(failed), "View"), stacklevel=2)
             views.extend(connection_referenced_views)
 
         # We need to include parent views in the schema to make sure that the schema is valid.
@@ -432,8 +424,9 @@ class DMSSchema:
                 if attr.name == "data_model":
                     if isinstance(items, list) and len(items) > 1:
                         warnings.warn(
-                            issues.importing.MultipleDataModelsWarning(
-                                [item.get("externalId", "Unknown") for item in items]
+                            MultipleResourcesWarning[str](
+                                frozenset([item.get("externalId", "Unknown") for item in items]),
+                                "DataModel",
                             ),
                             stacklevel=2,
                         )
@@ -522,8 +515,8 @@ class DMSSchema:
         else:
             raise ValueError(f"Cannot sort item of type {type(item)}")
 
-    def validate(self) -> list[DMSSchemaError]:
-        errors: set[DMSSchemaError] = set()
+    def validate(self) -> list[NeatError]:
+        errors: set[NeatError] = set()
         defined_spaces = self.spaces.copy()
         defined_containers = self.containers.copy()
         defined_views = self.views.copy()
@@ -535,28 +528,42 @@ class DMSSchema:
 
         for container in self.containers.values():
             if container.space not in defined_spaces:
-                errors.add(MissingSpaceError(space=container.space, referred_by=container.as_id()))
+                errors.add(
+                    ReferredResourceNotFoundError[str, dm.ContainerId](
+                        container.space, "Space", container.as_id(), "Container"
+                    )
+                )
 
         for view in self.views.values():
             view_id = view.as_id()
             if view.space not in defined_spaces:
-                errors.add(MissingSpaceError(space=view.space, referred_by=view_id))
+                errors.add(ReferredResourceNotFoundError[str, dm.ViewId](view.space, "Space", view_id, "View"))
 
             for parent in view.implements or []:
                 if parent not in defined_views:
-                    errors.add(MissingParentViewError(view=parent, referred_by=view_id))
+                    errors.add(
+                        ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
+                            parent, "View", view_id, "View", property_name="implements"
+                        )
+                    )
 
             for prop_name, prop in (view.properties or {}).items():
                 if isinstance(prop, dm.MappedPropertyApply):
                     ref_container = defined_containers.get(prop.container)
                     if ref_container is None:
-                        errors.add(MissingContainerError(container=prop.container, referred_by=view_id))
+                        errors.add(
+                            ReferredResourceNotFoundError[dm.ContainerId, dm.ViewId](
+                                prop.container, "Container", view_id, "View"
+                            )
+                        )
                     elif prop.container_property_identifier not in ref_container.properties:
                         errors.add(
-                            MissingContainerPropertyError(
-                                container=prop.container,
-                                property=prop.container_property_identifier,
-                                referred_by=view_id,
+                            ReferredPropertyNotFoundError[dm.ContainerId, dm.ViewId](
+                                prop.container,
+                                "Container",
+                                view_id,
+                                "View",
+                                property_name=prop.container_property_identifier,
                             )
                         )
                     else:
@@ -568,14 +575,22 @@ class DMSSchema:
                             )
 
                 if isinstance(prop, dm.EdgeConnectionApply) and prop.source not in defined_views:
-                    errors.add(MissingSourceViewError(view=prop.source, property=prop_name, referred_by=view_id))
+                    errors.add(
+                        ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
+                            prop.source, "View", view_id, "View", property_name=prop_name
+                        )
+                    )
 
                 if (
                     isinstance(prop, dm.EdgeConnectionApply)
                     and prop.edge_source is not None
                     and prop.edge_source not in defined_views
                 ):
-                    errors.add(MissingEdgeViewError(view=prop.edge_source, property=prop_name, referred_by=view_id))
+                    errors.add(
+                        ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
+                            prop.edge_source, "View", view_id, "View", property_name=prop_name
+                        )
+                    )
 
             # This allows for multiple view properties to be mapped to the same container property,
             # as long as they have different external_id, otherwise this will lead to raising
@@ -610,13 +625,21 @@ class DMSSchema:
         if self.data_model:
             model = self.data_model
             if model.space not in defined_spaces:
-                errors.add(MissingSpaceError(space=model.space, referred_by=model.as_id()))
+                errors.add(
+                    ReferredResourceNotFoundError[str, dm.DataModelId](
+                        model.space, "Space", model.as_id(), "Data Model"
+                    )
+                )
 
             view_counts: dict[dm.ViewId, int] = defaultdict(int)
             for view_id_or_class in model.views or []:
                 view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
                 if view_id not in defined_views:
-                    errors.add(MissingViewError(referred_by=model.as_id(), view=view_id))
+                    errors.add(
+                        ReferredResourceNotFoundError[dm.ViewId, dm.DataModelId](
+                            view_id, "View", model.as_id(), "DataModel"
+                        )
+                    )
                 view_counts[view_id] += 1
 
             for view_id, count in view_counts.items():

cognite/neat/rules/models/dms/_validation.py

@@ -3,8 +3,8 @@ from typing import Any, ClassVar
 
 from cognite.client import data_modeling as dm
 
+from cognite.neat.issues import IssueList, NeatIssueList
 from cognite.neat.rules import issues
-from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
 from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
 from cognite.neat.rules.models.data_types import DataType
@@ -31,7 +31,7 @@ class DMSPostValidation:
         self.views = rules.views
         self.issue_list = IssueList()
 
-    def validate(self) -> IssueList:
+    def validate(self) -> NeatIssueList:
         self._validate_raw_filter()
         self._consistent_container_properties()
 

cognite/neat/rules/models/entities.py

@@ -418,6 +418,9 @@ class ViewEntity(DMSVersionedEntity[ViewId]):
     ) -> ViewId:
         return ViewId(space=self.space, external_id=self.external_id, version=self.version)
 
+    def to_property_id(self, property_id: str) -> PropertyId:
+        return PropertyId(source=self.as_id(), property=property_id)
+
     @classmethod
     def from_id(cls, id: ViewId, default_version: str | None = None) -> "ViewEntity":
         if id.version is not None:

cognite/neat/rules/models/information/_validation.py

@@ -2,8 +2,8 @@ import itertools
 from collections import Counter
 from typing import cast
 
+from cognite.neat.issues import IssueList
 from cognite.neat.rules import issues
-from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
 from cognite.neat.utils.rdf_ import get_inheritance_path

cognite/neat/workflows/steps/lib/current/rules_importer.py

@@ -5,8 +5,8 @@ from typing import ClassVar
 from cognite.client import CogniteClient
 from cognite.client.data_classes.data_modeling import DataModelId
 
+from cognite.neat.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules import importers
-from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules.models import RoleTypes
 from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
 from cognite.neat.workflows._exceptions import StepNotInitialized
@@ -19,6 +19,7 @@ CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
 __all__ = [
     "ExcelToRules",
     "OntologyToRules",
+    "IMFToRules",
     "DMSToRules",
     "RulesInferenceFromRdfFile",
 ]
@@ -162,6 +163,77 @@ class OntologyToRules(Step):
         return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
 
 
+class IMFToRules(Step):
+    """This step import rules from the IMF-types and validates them."""
+
+    description = "This step imports rules from an RDF file "
+    version = "private-beta"
+    category = CATEGORY
+    configurables: ClassVar[list[Configurable]] = [
+        Configurable(
+            name="File name",
+            value="",
+            label="""Full file name of the RDF-file containing the IMF-types in the rules folder.
+                If not provided, step will attempt to get file name from payload
+                    of 'File Uploader' step (if exist)""",
+        ),
+        Configurable(
+            name="Report formatter",
+            value=next(iter(FORMATTER_BY_NAME.keys())),
+            label="The format of the report for the validation of the rules",
+            options=list(FORMATTER_BY_NAME),
+        ),
+        Configurable(
+            name="Role",
+            value="infer",
+            label="For what role Rules are intended?",
+            options=["infer", *RoleTypes.__members__.keys()],
+        ),
+    ]
+
+    def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData):  # type: ignore[syntax, override]
+        if self.configs is None or self.data_store_path is None:
+            raise StepNotInitialized(type(self).__name__)
+
+        file_name = self.configs.get("File name", None)
+
+        full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
+
+        if file_name:
+            rules_file_path = self.config.rules_store_path / file_name
+
+        elif full_path:
+            rules_file_path = full_path
+        else:
+            error_text = "Expected either 'File name' in the step config or 'File uploader' step uploading Excel Rules."
+            return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
+
+        # if role is None, it will be inferred from the rules file
+        role = self.configs.get("Role")
+        role_enum = None
+        if role != "infer" and role is not None:
+            role_enum = RoleTypes[role]
+
+        ontology_importer = importers.IMFImporter(filepath=rules_file_path)
+        rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
+
+        if rules is None:
+            output_dir = self.config.staging_path
+            report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
+            report_writer.write_to_file(issues, file_or_dir_path=output_dir)
+            report_file = report_writer.default_file_name
+            error_text = (
+                "<p></p>"
+                f'<a href="/data/staging/{report_file}?{time.time()}" '
+                f'target="_blank">Failed to validate rules, click here for report</a>'
+            )
+            return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
+
+        output_text = "Rules validation passed successfully!"
+
+        return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
+
+
 class DMSToRules(Step):
     """This step imports rules from CDF Data Model"""
 

cognite/neat/workflows/steps/lib/current/rules_validator.py

@@ -5,9 +5,9 @@ from typing import ClassVar
 
 from cognite.client import CogniteClient
 
-from cognite.neat.rules.issues import IssueList
-from cognite.neat.rules.issues.dms import MissingContainerError, MissingSpaceError, MissingViewError
-from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
+from cognite.neat.issues import NeatIssueList
+from cognite.neat.issues.errors.resources import ReferredResourceNotFoundError
+from cognite.neat.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules.models import DMSRules, SchemaCompleteness
 from cognite.neat.utils.cdf.loaders import ViewLoader
 from cognite.neat.workflows._exceptions import StepNotInitialized
@@ -61,9 +61,21 @@ class ValidateRulesAgainstCDF(Step):
         if not errors:
             return FlowMessage(output_text="Rules are complete and valid. No need to fetch from CDF.")
 
-        missing_spaces = [error.space for error in errors if isinstance(error, MissingSpaceError)]
-        missing_views = [error.view for error in errors if isinstance(error, MissingViewError)]
-        missing_containers = [error.container for error in errors if isinstance(error, MissingContainerError)]
+        missing_spaces = [
+            error.identifier
+            for error in errors
+            if isinstance(error, ReferredResourceNotFoundError) and error.resource_type == "Space"
+        ]
+        missing_views = [
+            error.identifier
+            for error in errors
+            if isinstance(error, ReferredResourceNotFoundError) and error.resource_type == "View"
+        ]
+        missing_containers = [
+            error.identifier
+            for error in errors
+            if isinstance(error, ReferredResourceNotFoundError) and error.resource_type == "Container"
+        ]
 
         retrieved_spaces = cdf_client.data_modeling.spaces.retrieve(missing_spaces).as_write()
         retrieved_containers = cdf_client.data_modeling.containers.retrieve(missing_containers).as_write()
@@ -87,7 +99,7 @@ class ValidateRulesAgainstCDF(Step):
             output_dir = self.data_store_path / Path("staging")
             report_writer = FORMATTER_BY_NAME[self.configs["Report Formatter"]]()
             report_writer.write_to_file(
-                IssueList(errors, title=dms_rules.metadata.name or dms_rules.metadata.external_id),
+                NeatIssueList(errors, title=dms_rules.metadata.name or dms_rules.metadata.external_id),
                 file_or_dir_path=output_dir,
             )
             report_file = report_writer.default_file_name

{cognite_neat-0.88.1.dist-info → cognite_neat-0.88.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.88.1
+Version: 0.88.2
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0