cognite-neat 0.99.0__py3-none-any.whl → 0.100.0__py3-none-any.whl

cognite/neat/_session/exceptions.py

@@ -2,6 +2,8 @@ import functools
 from collections.abc import Callable
 from typing import Any
 
+from cognite.neat._issues.errors import CDFMissingClientError
+
 from ._collector import _COLLECTOR
 
 try:
@@ -27,6 +29,8 @@ def _session_method_wrapper(func: Callable, cls_name: str):
         except NeatSessionError as e:
             action = _get_action()
             print(f"{_PREFIX} Cannot {action}: {e}")
+        except CDFMissingClientError as e:
+            print(f"{_PREFIX} {e.as_message()}")
         except ModuleNotFoundError as e:
             if e.name == "neatengine":
                 action = _get_action()

cognite/neat/_store/_base.py

@@ -174,34 +174,68 @@ class NeatGraphStore:
             self._add_triples(extractor.extract())
 
         if success:
-            self.provenance.append(
-                Change.record(
-                    activity=f"{type(extractor).__name__}",
-                    start=_start,
-                    end=datetime.now(timezone.utc),
-                    description=f"Extracted triples to graph store using {type(extractor).__name__}",
-                )
+            _end = datetime.now(timezone.utc)
+            # Need to do the hasattr in case the extractor comes from NeatEngine.
+            activities = (
+                extractor._get_activity_names()
+                if hasattr(extractor, "_get_activity_names")
+                else [type(extractor).__name__]
             )
+            for activity in activities:
+                self.provenance.append(
+                    Change.record(
+                        activity=activity,
+                        start=_start,
+                        end=_end,
+                        description=f"Extracted triples to graph store using {type(extractor).__name__}",
+                    )
+                )
 
-    def read(self, class_: str) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
-        """Read instances for given view from the graph store."""
-
-        if not self.rules:
+    def _read_via_rules_linkage(
+        self, class_neat_id: URIRef, property_link_pairs: dict[str, URIRef] | None
+    ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
+        if self.rules is None:
             warnings.warn("Rules not found in graph store!", stacklevel=2)
-            return None
+            return
+
+        if cls := InformationAnalysis(self.rules).classes_by_neat_id.get(class_neat_id):
+            if property_link_pairs:
+                property_renaming_config = {
+                    prop_uri: prop_name
+                    for prop_name, prop_neat_id in property_link_pairs.items()
+                    if (
+                        prop_uri := InformationAnalysis(self.rules).neat_id_to_transformation_property_uri(prop_neat_id)
+                    )
+                }
+
+                yield from self._read_via_class_entity(cls.class_, property_renaming_config)
+                return
+            else:
+                warnings.warn("Rules not linked", stacklevel=2)
+                return
+        else:
+            warnings.warn("Class with neat id {class_neat_id} found in rules", stacklevel=2)
+            return
 
-        class_entity = ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_)
+    def _read_via_class_entity(
+        self,
+        class_entity: ClassEntity,
+        property_renaming_config: dict[URIRef, str] | None = None,
+    ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
+        if self.rules is None:
+            warnings.warn("Rules not found in graph store!", stacklevel=2)
+            return
 
         if class_entity not in [definition.class_ for definition in self.rules.classes]:
             warnings.warn("Desired type not found in graph!", stacklevel=2)
-            return None
+            return
 
         if not (class_uri := InformationAnalysis(self.rules).class_uri(class_entity)):
             warnings.warn(
-                f"Class {class_} does not have namespace defined for prefix {class_entity.prefix} Rules!",
+                f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
                 stacklevel=2,
             )
-            return None
+            return
 
         has_hop_transformations = InformationAnalysis(self.rules).has_hop_transformations()
         has_self_reference_transformations = InformationAnalysis(
@@ -221,13 +255,15 @@ class NeatGraphStore:
                 msg,
                 stacklevel=2,
             )
-            return None
+            return
 
         # get all the instances for give class_uri
         instance_ids = self.queries.list_instances_ids_of_class(class_uri)
 
         # get potential property renaming config
-        property_renaming_config = InformationAnalysis(self.rules).define_property_renaming_config(class_entity)
+        property_renaming_config = property_renaming_config or InformationAnalysis(
+            self.rules
+        ).define_property_renaming_config(class_entity)
 
         # get property types to guide process of removing or not namespaces from results
         property_types = InformationAnalysis(self.rules).property_types(class_entity)
@@ -235,12 +271,36 @@ class NeatGraphStore:
         for instance_id in instance_ids:
             if res := self.queries.describe(
                 instance_id=instance_id,
-                instance_type=class_,
+                instance_type=class_entity.suffix,
                 property_renaming_config=property_renaming_config,
                 property_types=property_types,
             ):
                 yield res
 
+    def read(
+        self,
+        class_: str,
+    ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
+        """Read instances for given class from the graph store.
+
+        !!! note "Assumption"
+            This method assumes that the class_ belongs to the same (name)space as
+            the rules which are attached to the graph store.
+
+        """
+
+        if not self.rules:
+            warnings.warn("Rules not found in graph store!", stacklevel=2)
+            return
+
+        class_entity = ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_)
+
+        if class_entity not in [definition.class_ for definition in self.rules.classes]:
+            warnings.warn("Desired type not found in graph!", stacklevel=2)
+            return
+
+        yield from self._read_via_class_entity(class_entity)
+
     def _parse_file(
         self,
         filepath: Path,

cognite/neat/_utils/collection_.py

@@ -2,6 +2,8 @@ from collections import Counter
 from collections.abc import Iterable, Sequence
 from typing import TypeVar
 
+from cognite.neat._constants import IN_PYODIDE
+
 T_Element = TypeVar("T_Element")
 
 
@@ -21,3 +23,23 @@ def chunker(sequence: Sequence[T_Element], chunk_size: int) -> Iterable[Sequence
 
 def remove_list_elements(input_list: list, elements_to_remove: list) -> list:
     return [element for element in input_list if element not in elements_to_remove]
+
+
+def iterate_progress_bar(iterable: Iterable[T_Element], total: int, description: str) -> Iterable[T_Element]:
+    if IN_PYODIDE:
+        try:
+            from tqdm import tqdm  # type: ignore [import]
+        except ModuleNotFoundError:
+            return iterable
+        return tqdm(iterable, total=total, desc=description)
+    # Progress bar from rich requires multi-threading, which is not supported in Pyodide
+    try:
+        from rich.progress import track
+    except ModuleNotFoundError:
+        return iterable
+
+    return track(
+        iterable,
+        total=total,
+        description=description,
+    )

cognite/neat/_utils/rdf_.py

@@ -115,13 +115,15 @@ def as_neat_compliant_uri(uri: URIRef) -> URIRef:
     return URIRef(f"{namespace}{compliant_uri}")
 
 
-def convert_rdflib_content(content: RdfLiteral | URIRef | dict | list) -> Any:
-    if isinstance(content, RdfLiteral) or isinstance(content, URIRef):
+def convert_rdflib_content(content: RdfLiteral | URIRef | dict | list, remove_namespace: bool = False) -> Any:
+    if isinstance(content, RdfLiteral):
         return content.toPython()
+    elif isinstance(content, URIRef):
+        return remove_namespace_from_uri(content) if remove_namespace else content.toPython()
     elif isinstance(content, dict):
-        return {key: convert_rdflib_content(value) for key, value in content.items()}
+        return {key: convert_rdflib_content(value, remove_namespace) for key, value in content.items()}
     elif isinstance(content, list):
-        return [convert_rdflib_content(item) for item in content]
+        return [convert_rdflib_content(item, remove_namespace) for item in content]
     else:
         return content
 
@@ -206,6 +208,30 @@ def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: in
     check_commit(force_commit=True)
 
 
+def remove_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: int = 10_000) -> None:
+    """Removes triples from the graph store in batches.
+
+    Args:
+        triples: list of triples to be removed from the graph store
+        batch_size: Batch size of triples per commit, by default 10_000
+    """
+    batch_count = 0
+
+    def check_commit(force_commit: bool = False):
+        """Commit nodes to the graph if batch counter is reached or if force_commit is True"""
+        nonlocal batch_count
+        batch_count += 1
+        if force_commit or batch_count >= batch_size:
+            graph.commit()
+            batch_count = 0
+            return
+
+    for triple in triples:
+        graph.remove(triple)
+        check_commit()
+    check_commit(force_commit=True)
+
+
 def remove_instance_ids_in_batch(graph: Graph, instance_ids: Iterable[URIRef], batch_size: int = 1_000) -> None:
     """Removes all triples related to the given instances in the graph store in batches.
 

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.99.0"
-__engine__ = "^1.0.3"
+__version__ = "0.100.0"
+__engine__ = "^2.0.1"

cognite/neat/_workflows/steps/lib/current/rules_exporter.py

@@ -21,7 +21,6 @@ __all__ = [
     "RulesToOntology",
     "RulesToSHACL",
     "RulesToSemanticDataModel",
-    "RulesToCDFTransformations",
     "DeleteDataModelFromCDF",
 ]
 
@@ -68,7 +67,7 @@ class DeleteDataModelFromCDF(Step):
         if self.configs is None or self.data_store_path is None:
             raise WorkflowStepNotInitializedError(type(self).__name__)
         components_to_delete = {
-            cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
+            cast(Literal["spaces", "data_models", "views", "containers"], key)
             for key, value in self.complex_configs["Components"].items()
             if value
         }
@@ -181,7 +180,7 @@ class RulesToDMS(Step):
         )
         multi_space_components_create: bool = self.configs["Multi-space components create"] == "True"
         components_to_create = {
-            cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
+            cast(Literal["spaces", "data_models", "views", "containers"], key)
             for key, value in self.complex_configs["Components"].items()
             if value
         }
@@ -209,7 +208,7 @@ class RulesToDMS(Step):
         dms_exporter = exporters.DMSExporter(
             export_components=frozenset(components_to_create),
             include_space=(None if multi_space_components_create else {dms_rules.metadata.space}),
-            existing_handling=existing_components_handling,
+            existing=existing_components_handling,
         )
 
         output_dir = self.config.staging_path
@@ -532,93 +531,6 @@ class RulesToSemanticDataModel(Step):
         return FlowMessage(output_text=output_text)
 
 
-class RulesToCDFTransformations(Step):
-    description = "This step exports transformations and RAW tables to populate a data model in CDF"
-    version = "private-alpha"
-    category = CATEGORY
-    configurables: ClassVar[list[Configurable]] = [
-        Configurable(
-            name="Dry run",
-            value="False",
-            label=("Whether to perform a dry run of the export. "),
-            options=["True", "False"],
-        ),
-        Configurable(
-            name="Instance space",
-            value="",
-            label=(
-                "The space to use for the transformations instances. If provided, "
-                "the transformations will be set to populate"
-                "this space. If not provided, the space from the input rules will be used."
-            ),
-        ),
-    ]
-
-    def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage:  # type: ignore[override]
-        if self.configs is None or self.data_store_path is None:
-            raise WorkflowStepNotInitializedError(type(self).__name__)
-
-        input_rules = rules.dms or rules.information
-        if input_rules is None:
-            return FlowMessage(
-                error_text="Missing DMS or Information rules in the input data! "
-                "Please ensure that a DMS or Information rules is provided!",
-                step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
-            )
-        if isinstance(input_rules, DMSRules):
-            dms_rules = input_rules
-        elif isinstance(input_rules, InformationRules):
-            dms_rules = InformationToDMS().transform(input_rules).rules
-        else:
-            raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
-
-        instance_space = self.configs.get("Instance space") or dms_rules.metadata.space
-        dry_run = self.configs.get("Dry run", "False") == "True"
-        dms_exporter = exporters.DMSExporter(
-            export_pipeline=True, instance_space=instance_space, export_components=["spaces"]
-        )
-        output_dir = self.config.staging_path
-        output_dir.mkdir(parents=True, exist_ok=True)
-        file_name = dms_rules.metadata.external_id.replace(":", "_")
-        schema_zip = f"{file_name}_pipeline.zip"
-        schema_full_path = output_dir / schema_zip
-
-        dms_exporter.export_to_file(dms_rules, schema_full_path)
-
-        report_lines = ["# DMS Schema Export to CDF\n\n"]
-        errors = []
-        for result in dms_exporter.export_to_cdf_iterable(
-            rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run
-        ):
-            report_lines.append(str(result))
-            errors.extend(result.error_messages)
-
-        report_lines.append("\n\n# ERRORS\n\n")
-        report_lines.extend(errors)
-
-        output_dir = self.config.staging_path
-        output_dir.mkdir(parents=True, exist_ok=True)
-        report_file = "pipeline_creation_report.txt"
-        report_full_path = output_dir / report_file
-        report_full_path.write_text("\n".join(report_lines))
-
-        output_text = (
-            "<p></p>"
-            "Download Pipeline Export "
-            f'<a href="/data/staging/{report_file}?{time.time()}" '
-            f'target="_blank">Report</a>'
-            "<p></p>"
-            "Download Pipeline exported schema"
-            f'- <a href="/data/staging/{schema_zip}?{time.time()}" '
-            f'target="_blank">{schema_zip}</a>'
-        )
-
-        if errors:
-            return FlowMessage(error_text=output_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
-        else:
-            return FlowMessage(output_text=output_text)
-
-
 def _get_default_file_name(rules: MultiRuleData, file_category: str = "ontology", extension: str = "ttl") -> str:
     name = rules.information.metadata.prefix if rules.information else cast(DMSRules, rules.dms).metadata.space
     version = rules.information.metadata.version if rules.information else cast(DMSRules, rules.dms).metadata.version

cognite/neat/_workflows/steps/lib/current/rules_importer.py

@@ -3,7 +3,6 @@ from pathlib import Path
 from typing import ClassVar
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes.data_modeling import DataModelId
 
 from cognite.neat._client import NeatClient
 from cognite.neat._issues.errors import WorkflowStepNotInitializedError
@@ -288,21 +287,8 @@ class DMSToRules(Step):
                 f"or 'my_space:my_data_model', failed to parse space from {datamodel_id_str}"
             )
             return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
-        ref_datamodel_str = self.configs.get("Reference data model id", "")
-        ref_model_id: DataModelId | None = None
-        if ref_datamodel_str:
-            ref_model = DataModelEntity.load(ref_datamodel_str)
-            if isinstance(ref_model, DMSUnknownEntity):
-                error_text = (
-                    f"Reference data model id should be in the format 'my_space:my_data_model(version=1)' "
-                    f"or 'my_space:my_data_model', failed to parse space from {ref_datamodel_str}"
-                )
-                return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
-            ref_model_id = ref_model.as_id()
-
-        dms_importer = importers.DMSImporter.from_data_model_id(
-            NeatClient(cdf_client), datamodel_entity.as_id(), ref_model_id
-        )
+
+        dms_importer = importers.DMSImporter.from_data_model_id(NeatClient(cdf_client), datamodel_entity.as_id())
 
         # if role is None, it will be inferred from the rules file
         role = self.configs.get("Role")

cognite/neat/_workflows/steps/lib/current/rules_validator.py

@@ -11,6 +11,7 @@ from cognite.neat._issues import NeatIssueList
 from cognite.neat._issues.errors import ResourceNotFoundError, WorkflowStepNotInitializedError
 from cognite.neat._issues.formatters import FORMATTER_BY_NAME
 from cognite.neat._rules.models import DMSRules
+from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat._workflows.steps.data_contracts import MultiRuleData
 from cognite.neat._workflows.steps.step_model import Configurable, Step
@@ -51,10 +52,7 @@ class ValidateRulesAgainstCDF(Step):
             )
         dms_rules = rules.dms
 
-        schema = dms_rules.as_schema()
-        errors = schema.validate()
-        if not errors:
-            return FlowMessage(output_text="Rules are complete and valid. No need to fetch from CDF.")
+        errors = DMSValidation(dms_rules, NeatClient(cdf_client)).validate()
 
         missing_spaces = [
             error.identifier
@@ -85,11 +83,11 @@ class ValidateRulesAgainstCDF(Step):
             f"and {len(retrieved_views)} views from CDF."
         )
 
+        schema = dms_rules.as_schema()
         schema.spaces.update({space.space: space for space in retrieved_spaces})
         schema.containers.update({container.as_id(): container for container in retrieved_containers})
         schema.views.update({view.as_id(): view for view in retrieved_views})
 
-        errors = schema.validate()
         if errors:
             output_dir = self.data_store_path / Path("staging")
             report_writer = FORMATTER_BY_NAME[self.configs["Report Formatter"]]()

{cognite_neat-0.99.0.dist-info → cognite_neat-0.100.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.99.0
+Version: 0.100.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0