cognite-neat 0.99.0__py3-none-any.whl → 0.99.1__py3-none-any.whl

cognite/neat/_rules/models/information/_rules.py

@@ -1,5 +1,4 @@
 import math
-import sys
 from collections.abc import Hashable
 from typing import TYPE_CHECKING, Any, ClassVar
 
@@ -34,8 +33,6 @@ from cognite.neat._rules.models.entities import (
     ClassEntity,
     ClassEntityList,
     Entity,
-    MultiValueTypeInfo,
-    Undefined,
     UnknownEntity,
 )
 
@@ -43,12 +40,6 @@ if TYPE_CHECKING:
     from cognite.neat._rules.models import DMSRules
 
 
-if sys.version_info >= (3, 11):
-    from typing import Self
-else:
-    from typing_extensions import Self
-
-
 class InformationMetadata(BaseMetadata):
     role: ClassVar[RoleTypes] = RoleTypes.information
     aspect: ClassVar[DataModelAspect] = DataModelAspect.logical
@@ -155,21 +146,6 @@ class InformationProperty(SheetRow):
         else:
             raise NeatValueError(f"Invalid RDF Path: {value!s}")
 
-    @model_validator(mode="after")
-    def set_default_as_list(self):
-        if (
-            self.type_ == EntityTypes.data_property
-            and self.default
-            and self.is_list
-            and not isinstance(self.default, list)
-        ):
-            if isinstance(self.default, str):
-                if self.default:
-                    self.default = self.default.replace(", ", ",").split(",")
-                else:
-                    self.default = [self.default]
-        return self
-
     @model_validator(mode="after")
     def set_type_for_default(self):
         if self.type_ == EntityTypes.data_property and self.default:
@@ -243,41 +219,6 @@ class InformationRules(BaseRules):
             values = get_default_prefixes()
         return values
 
-    @model_validator(mode="after")
-    def update_entities_prefix(self) -> Self:
-        # update expected_value_types
-        for property_ in self.properties:
-            if isinstance(property_.value_type, ClassEntity) and property_.value_type.prefix is Undefined:
-                property_.value_type.prefix = self.metadata.prefix
-
-            if isinstance(property_.value_type, MultiValueTypeInfo):
-                property_.value_type.set_default_prefix(self.metadata.prefix)
-
-            if property_.class_.prefix is Undefined:
-                property_.class_.prefix = self.metadata.prefix
-
-        # update implements
-        for class_ in self.classes:
-            if class_.implements:
-                for parent in class_.implements:
-                    if not isinstance(parent.prefix, str):
-                        parent.prefix = self.metadata.prefix
-            if class_.class_.prefix is Undefined:
-                class_.class_.prefix = self.metadata.prefix
-
-        return self
-
-    @model_validator(mode="after")
-    def post_validation(self) -> "InformationRules":
-        from ._validation import InformationPostValidation
-
-        issue_list = InformationPostValidation(self).validate()
-        if issue_list.warnings:
-            issue_list.trigger_warnings()
-        if issue_list.has_errors:
-            raise issue_list.as_exception()
-        return self
-
     def as_dms_rules(self) -> "DMSRules":
         from cognite.neat._rules.transformers._converters import _InformationRulesConverter
 

cognite/neat/_rules/models/information/_validation.py

@@ -1,17 +1,17 @@
 import itertools
 from collections import Counter
-from typing import cast
 
 from cognite.neat._issues import IssueList
-from cognite.neat._issues.errors import NeatValueError, ResourceNotDefinedError
+from cognite.neat._issues.errors import NeatValueError
 from cognite.neat._issues.warnings._models import UndefinedClassWarning
+from cognite.neat._issues.warnings._resources import ResourceNotDefinedWarning
 from cognite.neat._rules._constants import EntityTypes
 from cognite.neat._rules.models.entities import ClassEntity, UnknownEntity
 
 from ._rules import InformationRules
 
 
-class InformationPostValidation:
+class InformationValidation:
     """This class does all the validation of the Information rules that have dependencies
     between components."""
 
@@ -44,10 +44,10 @@ class InformationPostValidation:
                 # same prefix, meaning same space
                 if not class_parent_pairs[class_] and class_.prefix == self.metadata.prefix:
                     self.issue_list.append(
-                        ResourceNotDefinedError[ClassEntity](
+                        ResourceNotDefinedWarning(
                             resource_type="class",
                             identifier=class_,
-                            location="Classes sheet",
+                            location="Properties sheet",
                         )
                     )
 
@@ -63,7 +63,7 @@ class InformationPostValidation:
                     self.issue_list.append(UndefinedClassWarning(class_id=str(parent)))
                 else:
                     self.issue_list.append(
-                        ResourceNotDefinedError[ClassEntity](
+                        ResourceNotDefinedWarning(
                             resource_type="class",
                             identifier=parent,
                             location="Classes sheet",
@@ -79,7 +79,7 @@ class InformationPostValidation:
         if missing_classes := classes_with_explicit_properties.difference(defined_classes):
             for class_ in missing_classes:
                 self.issue_list.append(
-                    ResourceNotDefinedError[ClassEntity](
+                    ResourceNotDefinedWarning(
                         resource_type="class",
                         identifier=class_,
                         location="Classes sheet",
@@ -99,9 +99,9 @@ class InformationPostValidation:
             # Todo: include row and column number
             for missing in missing_value_types:
                 self.issue_list.append(
-                    ResourceNotDefinedError[ClassEntity](
+                    ResourceNotDefinedWarning(
                         resource_type="class",
-                        identifier=cast(ClassEntity, missing),
+                        identifier=missing,
                         location="Classes sheet",
                     )
                 )

cognite/neat/_rules/models/mapping/_classic2core.py

@@ -32,7 +32,7 @@ def load_classic_to_core_mapping(org_name: str, source_space: str, source_versio
     if not isinstance(read.rules, DMSInputRules):
         raise NeatValueError(f"Expected DMS rules, but got {type(read.rules).__name__}")
 
-    verified = VerifyDMSRules(errors="raise", post_validate=False).transform(read)
+    verified = VerifyDMSRules(errors="raise", validate=False).transform(read)
 
     if verified.rules is None:
         raise NeatValueError("Failed to verify the rules.")

cognite/neat/_rules/models/mapping/_classic2core.yaml

@@ -65,7 +65,7 @@ properties:
   index: source
   is_list: false
   nullable: true
-  value_type: cdf_cdm:SourceSystem(version=v1)
+  value_type: cdf_cdm:CogniteSourceSystem(version=v1)
   view: ClassicAsset
   view_property: source
 - container: cdf_cdm:CogniteDescribable
@@ -124,7 +124,7 @@ properties:
   index: source
   is_list: false
   nullable: true
-  value_type: cdf_cdm:SourceSystem(version=v1)
+  value_type: cdf_cdm:CogniteSourceSystem(version=v1)
   view: ClassicEvent
   view_property: source
 - container: cdf_cdm:CogniteSchedulable
@@ -196,7 +196,7 @@ properties:
   index: source
   is_list: false
   nullable: true
-  value_type: cdf_cdm:SourceSystem(version=v1)
+  value_type: cdf_cdm:CogniteSourceSystem(version=v1)
   view: ClassicFile
   view_property: source
 - container: cdf_cdm:CogniteSourceable
@@ -306,7 +306,7 @@ properties:
   is_list: false
   name: Unit
   nullable: true
-  value_type: Unit
+  value_type: cdf_cdm:CogniteUnit(version=v1)
   view: ClassicTimeSeries
   view_property: unitExternalId
 views:
@@ -315,6 +315,10 @@ views:
   implements: cdf_cdm:CogniteDescribable(version=v1)
   in_model: true
   view: cdf_cdm:CogniteSourceSystem(version=v1)
+- description: Represents a single unit of measurement
+  implements: CogniteDescribable
+  in_model: true
+  view: cdf_cdm:CogniteUnit(version=v1)
 - description: Assets represent systems that support industrial functions or processes.
     Assets are often called 'functional location'.
   implements: cdf_cdm:CogniteAsset(version=v1)

cognite/neat/_rules/transformers/_pipelines.py

@@ -23,7 +23,7 @@ class ImporterPipeline(RulesPipeline[InputRules, VerifiedRules]):
 
     @classmethod
     def _create_pipeline(cls, importer: BaseImporter[InputRules], role: RoleTypes | None = None) -> "ImporterPipeline":
-        items: list[RulesTransformer] = [VerifyAnyRules(errors="continue")]
+        items: list[RulesTransformer] = [VerifyAnyRules(errors="continue", validate=True)]
         if role is not None:
             out_cls = VERIFIED_RULES_BY_ROLE[role]
             items.append(ConvertToRules(out_cls))

cognite/neat/_rules/transformers/_verification.py

@@ -1,7 +1,7 @@
 from abc import ABC
 from typing import Any, Literal
 
-from cognite.neat._issues import IssueList, NeatError, NeatWarning, catch_issues
+from cognite.neat._issues import IssueList, MultiValueError, NeatError, NeatWarning, catch_issues
 from cognite.neat._issues.errors import NeatTypeError
 from cognite.neat._rules._shared import (
     InputRules,
@@ -18,6 +18,8 @@ from cognite.neat._rules.models import (
     InformationInputRules,
     InformationRules,
 )
+from cognite.neat._rules.models.dms import DMSValidation
+from cognite.neat._rules.models.information import InformationValidation
 
 from ._base import RulesTransformer
 
@@ -26,10 +28,11 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
     """Base class for all verification transformers."""
 
     _rules_cls: type[T_VerifiedRules]
+    _validation_cls: type
 
-    def __init__(self, errors: Literal["raise", "continue"], post_validate: bool = True) -> None:
+    def __init__(self, errors: Literal["raise", "continue"], validate: bool = True) -> None:
         self.errors = errors
-        self.post_validate = post_validate
+        self.validate = validate
 
     def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
         issues = IssueList()
@@ -41,8 +44,17 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
         with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
             rules_cls = self._get_rules_cls(in_)
             dumped = in_.dump()
-            dumped["post_validate"] = self.post_validate
             verified_rules = rules_cls.model_validate(dumped)  # type: ignore[assignment]
+            if self.validate:
+                validation_cls = self._get_validation_cls(verified_rules)  # type: ignore[arg-type]
+                validation_issues = validation_cls(verified_rules).validate()
+                # We need to trigger warnings are raise exceptions such that they are caught by the context manager
+                # and processed with the read context
+                if validation_issues.warnings:
+                    validation_issues.trigger_warnings()
+                if validation_issues.has_errors:
+                    verified_rules = None
+                    raise MultiValueError(validation_issues.errors)
 
         if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
             raise issues.as_errors()
@@ -54,17 +66,22 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
     def _get_rules_cls(self, in_: T_InputRules) -> type[T_VerifiedRules]:
         return self._rules_cls
 
+    def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
+        return self._validation_cls
+
 
 class VerifyDMSRules(VerificationTransformer[DMSInputRules, DMSRules]):
     """Class to verify DMS rules."""
 
     _rules_cls = DMSRules
+    _validation_cls = DMSValidation
 
 
 class VerifyInformationRules(VerificationTransformer[InformationInputRules, InformationRules]):
     """Class to verify Information rules."""
 
     _rules_cls = InformationRules
+    _validation_cls = InformationValidation
 
 
 class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
@@ -77,3 +94,11 @@ class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
             return DMSRules
         else:
             raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
+
+    def _get_validation_cls(self, rules: T_VerifiedRules) -> type:
+        if isinstance(rules, InformationRules):
+            return InformationValidation
+        elif isinstance(rules, DMSRules):
+            return DMSValidation
+        else:
+            raise NeatTypeError(f"Unsupported rules type: {type(rules)}")

cognite/neat/_session/_base.py

@@ -10,8 +10,9 @@ from cognite.neat._issues import IssueList, catch_issues
 from cognite.neat._issues.errors import RegexViolationError
 from cognite.neat._rules import importers
 from cognite.neat._rules._shared import ReadRules, VerifiedRules
-from cognite.neat._rules.importers import DMSImporter
-from cognite.neat._rules.models import DMSInputRules, DMSRules, SheetList
+from cognite.neat._rules.models import DMSRules
+from cognite.neat._rules.models.dms import DMSValidation
+from cognite.neat._rules.models.information import InformationValidation
 from cognite.neat._rules.models.information._rules import InformationRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
@@ -66,31 +67,21 @@ class NeatSession:
 
     def verify(self) -> IssueList:
         source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
-
-        reference_rules: DMSInputRules | None = None
-        if isinstance(last_unverified_rule.rules, DMSInputRules):
-            dms_rules = last_unverified_rule.rules
-            views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
-            if views_ids or containers_ids:
-                if self._client is None:
-                    raise NeatSessionError(
-                        "No client provided. You are referencing unknown views and containers in your data model, "
-                        "NEAT needs a client to lookup the definitions. "
-                        "Please set the client in the session, NeatSession(client=client)."
-                    )
-                schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
-
-                importer = DMSImporter(schema)
-                reference_rules = importer.to_rules().rules
-
-                if reference_rules is not None:
-                    dms_rules.views.extend(reference_rules.views)
-                    if dms_rules.containers:
-                        dms_rules.containers.extend(reference_rules.containers or [])
-
-        transformer = VerifyAnyRules("continue")
+        transformer = VerifyAnyRules("continue", validate=False)
         start = datetime.now(timezone.utc)
         output = transformer.try_transform(last_unverified_rule)
+        if isinstance(output.rules, DMSRules):
+            issues = DMSValidation(output.rules, self._client).validate()
+        elif isinstance(output.rules, InformationRules):
+            issues = InformationValidation(output.rules).validate()
+        else:
+            raise NeatSessionError("Unsupported rule type")
+        if issues.has_errors:
+            # This is up for discussion, but I think we should not return rules that
+            # only pass the verification but not the validation.
+            output.rules = None
+        output.issues.extend(issues)
+
         end = datetime.now(timezone.utc)
 
         if output.rules:
@@ -103,22 +94,6 @@ class NeatSession:
                 self._state.data_model.provenance.source_entity(source_id)
                 or self._state.data_model.provenance.target_entity(source_id),
             )
-            if reference_rules is not None and isinstance(output.rules, DMSRules):
-                # Remove the referenced views and containers from the rules
-                ref_view_ids = set(reference_rules.as_view_entities())
-                if ref_view_ids:
-                    output.rules.views = SheetList(
-                        [view for view in output.rules.views if view.view not in ref_view_ids]
-                    )
-                ref_container_ids = reference_rules.as_container_entities()
-                if output.rules.containers and ref_container_ids:
-                    output.rules.containers = SheetList(
-                        [
-                            container
-                            for container in output.rules.containers
-                            if container.container not in ref_container_ids
-                        ]
-                    )
 
             self._state.data_model.write(output.rules, change)
 

cognite/neat/_session/_prepare.py

@@ -13,6 +13,7 @@ from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
 from cognite.neat._rules._shared import InputRules, ReadRules
 from cognite.neat._rules.importers import DMSImporter
 from cognite.neat._rules.models import DMSRules
+from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 from cognite.neat._rules.transformers import (
     PrefixEntities,
@@ -316,7 +317,7 @@ class DataModelPrepareAPI:
         source_id, rules = self._state.data_model.last_verified_dms_rules
 
         dms_ref: DMSRules | None = None
-        view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
+        view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
         if view_ids or container_ids:
             if self._client is None:
                 raise NeatSessionError(
@@ -324,7 +325,7 @@ class DataModelPrepareAPI:
                     "NEAT needs a client to lookup the definitions. "
                     "Please set the client in the session, NeatSession(client=client)."
                 )
-            schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
+            schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
 
             importer = DMSImporter(schema)
             reference_rules = importer.to_rules().rules
@@ -409,7 +410,7 @@ class DataModelPrepareAPI:
         start = datetime.now(timezone.utc)
 
         source_id, rules = self._state.data_model.last_verified_dms_rules
-        view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
+        view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
         if not (view_ids or container_ids):
             print(
                 f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
@@ -422,7 +423,7 @@ class DataModelPrepareAPI:
                 "NEAT needs a client to lookup the definitions. "
                 "Please set the client in the session, NeatSession(client=client)."
             )
-        schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
+        schema = self._client.schema.retrieve([v.as_id() for v in view_ids], [c.as_id() for c in container_ids])
         copy_ = rules.model_copy(deep=True)
         copy_.metadata.version = f"{rules.metadata.version}_completed"
         importer = DMSImporter(schema)
@@ -433,7 +434,7 @@ class DataModelPrepareAPI:
                 "Could not import the referenced views and containers. "
                 "See `neat.inspect.issues()` for more information."
             )
-        verified = VerifyDMSRules("continue", post_validate=False).transform(imported.rules)
+        verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
         if verified.rules is None:
             self._state.data_model.issue_lists.append(verified.issues)
             raise NeatSessionError(

cognite/neat/_session/_to.py

@@ -66,9 +66,12 @@ class ToAPI:
                     "This is required for the 'toolkit' format."
                 )
             dms_rule = self._state.data_model.last_verified_dms_rules[1]
-            exporters.DMSExporter().export_to_file(dms_rule, Path(io))
+            user_path = Path(io)
+            if user_path.suffix == "" and not user_path.exists():
+                user_path.mkdir(parents=True)
+            exporters.DMSExporter().export_to_file(dms_rule, user_path)
         else:
-            raise NeatSessionError("Please provide a valid format. {['neat', 'toolkit']}")
+            raise NeatSessionError("Please provide a valid format. 'neat' or 'toolkit'")
 
         return None
 

cognite/neat/_session/exceptions.py

@@ -2,6 +2,8 @@ import functools
 from collections.abc import Callable
 from typing import Any
 
+from cognite.neat._issues.errors import CDFMissingClientError
+
 from ._collector import _COLLECTOR
 
 try:
@@ -27,6 +29,8 @@ def _session_method_wrapper(func: Callable, cls_name: str):
         except NeatSessionError as e:
             action = _get_action()
             print(f"{_PREFIX} Cannot {action}: {e}")
+        except CDFMissingClientError as e:
+            print(f"{_PREFIX} {e.as_message()}")
         except ModuleNotFoundError as e:
             if e.name == "neatengine":
                 action = _get_action()

cognite/neat/_utils/rdf_.py

@@ -115,13 +115,15 @@ def as_neat_compliant_uri(uri: URIRef) -> URIRef:
     return URIRef(f"{namespace}{compliant_uri}")
 
 
-def convert_rdflib_content(content: RdfLiteral | URIRef | dict | list) -> Any:
-    if isinstance(content, RdfLiteral) or isinstance(content, URIRef):
+def convert_rdflib_content(content: RdfLiteral | URIRef | dict | list, remove_namespace: bool = False) -> Any:
+    if isinstance(content, RdfLiteral):
         return content.toPython()
+    elif isinstance(content, URIRef):
+        return remove_namespace_from_uri(content) if remove_namespace else content.toPython()
     elif isinstance(content, dict):
-        return {key: convert_rdflib_content(value) for key, value in content.items()}
+        return {key: convert_rdflib_content(value, remove_namespace) for key, value in content.items()}
     elif isinstance(content, list):
-        return [convert_rdflib_content(item) for item in content]
+        return [convert_rdflib_content(item, remove_namespace) for item in content]
     else:
         return content
 

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.99.0"
+__version__ = "0.99.1"
 __engine__ = "^1.0.3"

cognite/neat/_workflows/steps/lib/current/rules_exporter.py

@@ -21,7 +21,6 @@ __all__ = [
     "RulesToOntology",
     "RulesToSHACL",
     "RulesToSemanticDataModel",
-    "RulesToCDFTransformations",
     "DeleteDataModelFromCDF",
 ]
 
@@ -532,93 +531,6 @@ class RulesToSemanticDataModel(Step):
         return FlowMessage(output_text=output_text)
 
 
-class RulesToCDFTransformations(Step):
-    description = "This step exports transformations and RAW tables to populate a data model in CDF"
-    version = "private-alpha"
-    category = CATEGORY
-    configurables: ClassVar[list[Configurable]] = [
-        Configurable(
-            name="Dry run",
-            value="False",
-            label=("Whether to perform a dry run of the export. "),
-            options=["True", "False"],
-        ),
-        Configurable(
-            name="Instance space",
-            value="",
-            label=(
-                "The space to use for the transformations instances. If provided, "
-                "the transformations will be set to populate"
-                "this space. If not provided, the space from the input rules will be used."
-            ),
-        ),
-    ]
-
-    def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage:  # type: ignore[override]
-        if self.configs is None or self.data_store_path is None:
-            raise WorkflowStepNotInitializedError(type(self).__name__)
-
-        input_rules = rules.dms or rules.information
-        if input_rules is None:
-            return FlowMessage(
-                error_text="Missing DMS or Information rules in the input data! "
-                "Please ensure that a DMS or Information rules is provided!",
-                step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
-            )
-        if isinstance(input_rules, DMSRules):
-            dms_rules = input_rules
-        elif isinstance(input_rules, InformationRules):
-            dms_rules = InformationToDMS().transform(input_rules).rules
-        else:
-            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
-
-        instance_space = self.configs.get("Instance space") or dms_rules.metadata.space
-        dry_run = self.configs.get("Dry run", "False") == "True"
-        dms_exporter = exporters.DMSExporter(
-            export_pipeline=True, instance_space=instance_space, export_components=["spaces"]
-        )
-        output_dir = self.config.staging_path
-        output_dir.mkdir(parents=True, exist_ok=True)
-        file_name = dms_rules.metadata.external_id.replace(":", "_")
-        schema_zip = f"{file_name}_pipeline.zip"
-        schema_full_path = output_dir / schema_zip
-
-        dms_exporter.export_to_file(dms_rules, schema_full_path)
-
-        report_lines = ["# DMS Schema Export to CDF\n\n"]
-        errors = []
-        for result in dms_exporter.export_to_cdf_iterable(
-            rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run
-        ):
-            report_lines.append(str(result))
-            errors.extend(result.error_messages)
-
-        report_lines.append("\n\n# ERRORS\n\n")
-        report_lines.extend(errors)
-
-        output_dir = self.config.staging_path
-        output_dir.mkdir(parents=True, exist_ok=True)
-        report_file = "pipeline_creation_report.txt"
-        report_full_path = output_dir / report_file
-        report_full_path.write_text("\n".join(report_lines))
-
-        output_text = (
-            "<p></p>"
-            "Download Pipeline Export "
-            f'<a href="/data/staging/{report_file}?{time.time()}" '
-            f'target="_blank">Report</a>'
-            "<p></p>"
-            "Download Pipeline exported schema"
-            f'- <a href="/data/staging/{schema_zip}?{time.time()}" '
-            f'target="_blank">{schema_zip}</a>'
-        )
-
-        if errors:
-            return FlowMessage(error_text=output_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
-        else:
-            return FlowMessage(output_text=output_text)
-
-
 def _get_default_file_name(rules: MultiRuleData, file_category: str = "ontology", extension: str = "ttl") -> str:
     name = rules.information.metadata.prefix if rules.information else cast(DMSRules, rules.dms).metadata.space
     version = rules.information.metadata.version if rules.information else cast(DMSRules, rules.dms).metadata.version

cognite/neat/_workflows/steps/lib/current/rules_importer.py

@@ -3,7 +3,6 @@ from pathlib import Path
 from typing import ClassVar
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes.data_modeling import DataModelId
 
 from cognite.neat._client import NeatClient
 from cognite.neat._issues.errors import WorkflowStepNotInitializedError
@@ -288,21 +287,8 @@ class DMSToRules(Step):
                 f"or 'my_space:my_data_model', failed to parse space from {datamodel_id_str}"
             )
             return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
-        ref_datamodel_str = self.configs.get("Reference data model id", "")
-        ref_model_id: DataModelId | None = None
-        if ref_datamodel_str:
-            ref_model = DataModelEntity.load(ref_datamodel_str)
-            if isinstance(ref_model, DMSUnknownEntity):
-                error_text = (
-                    f"Reference data model id should be in the format 'my_space:my_data_model(version=1)' "
-                    f"or 'my_space:my_data_model', failed to parse space from {ref_datamodel_str}"
-                )
-                return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
-            ref_model_id = ref_model.as_id()
-
-        dms_importer = importers.DMSImporter.from_data_model_id(
-            NeatClient(cdf_client), datamodel_entity.as_id(), ref_model_id
-        )
+
+        dms_importer = importers.DMSImporter.from_data_model_id(NeatClient(cdf_client), datamodel_entity.as_id())
 
         # if role is None, it will be inferred from the rules file
         role = self.configs.get("Role")

cognite/neat/_workflows/steps/lib/current/rules_validator.py

@@ -11,6 +11,7 @@ from cognite.neat._issues import NeatIssueList
 from cognite.neat._issues.errors import ResourceNotFoundError, WorkflowStepNotInitializedError
 from cognite.neat._issues.formatters import FORMATTER_BY_NAME
 from cognite.neat._rules.models import DMSRules
+from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat._workflows.steps.data_contracts import MultiRuleData
 from cognite.neat._workflows.steps.step_model import Configurable, Step
@@ -51,10 +52,7 @@ class ValidateRulesAgainstCDF(Step):
             )
         dms_rules = rules.dms
 
-        schema = dms_rules.as_schema()
-        errors = schema.validate()
-        if not errors:
-            return FlowMessage(output_text="Rules are complete and valid. No need to fetch from CDF.")
+        errors = DMSValidation(dms_rules, NeatClient(cdf_client)).validate()
 
         missing_spaces = [
             error.identifier
@@ -85,11 +83,11 @@ class ValidateRulesAgainstCDF(Step):
             f"and {len(retrieved_views)} views from CDF."
         )
 
+        schema = dms_rules.as_schema()
         schema.spaces.update({space.space: space for space in retrieved_spaces})
         schema.containers.update({container.as_id(): container for container in retrieved_containers})
         schema.views.update({view.as_id(): view for view in retrieved_views})
 
-        errors = schema.validate()
         if errors:
             output_dir = self.data_store_path / Path("staging")
             report_writer = FORMATTER_BY_NAME[self.configs["Report Formatter"]]()