cognite-neat 0.87.0__py3-none-any.whl → 0.87.3__py3-none-any.whl

cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py}

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
 from collections.abc import (
+    Hashable,
     ItemsView,
     Iterable,
     Iterator,
@@ -8,11 +9,18 @@ from collections.abc import (
     MutableMapping,
     ValuesView,
 )
+from dataclasses import dataclass
 from typing import Any, TypeVar, cast, final
 
 import pandas as pd
 import yaml
-from cognite.client.data_classes._base import T_CogniteResource
+from cognite.client import CogniteClient
+from cognite.client.data_classes._base import (
+    CogniteResourceList,
+    T_CogniteResource,
+    WriteableCogniteResource,
+    WriteableCogniteResourceList,
+)
 from cognite.client.data_classes.data_modeling import (
     ContainerApply,
     ContainerId,
@@ -29,7 +37,119 @@ from cognite.client.utils._pandas_helpers import (
     convert_nullable_int_cols,
 )
 
-T_ID = TypeVar("T_ID")
+# The Table, TableWrite data classes in the Cognite-SDK lacks the database attribute.
+# This is a problem when creating the RawTableLoader that needs the data class to be able to create, update, retrieve
+# and delete tables.
+# This is a reimplemented version of the Table, TableWrite data classes with the database attribute added.
+
+
+@dataclass(frozen=True)
+class RawTableID:
+    table: str
+    database: str
+
+    def as_tuple(self) -> tuple[str, str]:
+        return self.database, self.table
+
+
+class RawTableCore(WriteableCogniteResource["RawTableWrite"], ABC):
+    """A NoSQL database table to store customer data
+
+    Args:
+        name (str | None): Unique name of the table
+    """
+
+    def __init__(
+        self,
+        name: str | None = None,
+        database: str | None = None,
+    ) -> None:
+        self.name = name
+        self.database = database
+
+    def as_id(self) -> RawTableID:
+        if self.name is None or self.database is None:
+            raise ValueError("name and database are required to create a TableID")
+        return RawTableID(table=self.name, database=self.database)
+
+
+class RawTable(RawTableCore):
+    """A NoSQL database table to store customer data.
+    This is the reading version of the Table class, which is used when retrieving a table.
+
+    Args:
+        name (str | None): Unique name of the table
+        created_time (int | None): Time the table was created.
+        cognite_client (CogniteClient | None): The client to associate with this object.
+    """
+
+    def __init__(
+        self,
+        name: str | None = None,
+        database: str | None = None,
+        created_time: int | None = None,
+        cognite_client: CogniteClient | None = None,
+    ) -> None:
+        super().__init__(name, database)
+        self.created_time = created_time
+        self._cognite_client = cast("CogniteClient", cognite_client)
+
+        self._db_name: str | None = None
+
+    def as_write(self) -> "RawTableWrite":
+        """Returns this Table as a TableWrite"""
+        if self.name is None or self.database is None:
+            raise ValueError("name and database are required to create a Table")
+        return RawTableWrite(name=self.name, database=self.database)
+
+
+class RawTableWrite(RawTableCore):
+    """A NoSQL database table to store customer data
+    This is the writing version of the Table class, which is used when creating a table.
+
+    Args:
+        name (str): Unique name of the table
+    """
+
+    def __init__(
+        self,
+        name: str,
+        database: str,
+    ) -> None:
+        super().__init__(name, database)
+
+    @classmethod
+    def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> "RawTableWrite":
+        return cls(resource["name"], resource["database"])
+
+    def as_write(self) -> "RawTableWrite":
+        """Returns this TableWrite instance."""
+        return self
+
+
+class RawTableWriteList(CogniteResourceList[RawTableWrite]):
+    _RESOURCE = RawTableWrite
+
+    def as_ids(self) -> list[RawTableID]:
+        """Returns this TableWriteList as a list of TableIDs"""
+        return [table.as_id() for table in self.data]
+
+
+class RawTableList(
+    WriteableCogniteResourceList[RawTableWrite, RawTable],
+):
+    _RESOURCE = RawTable
+
+    def as_write(self) -> RawTableWriteList:
+        """Returns this TableList as a TableWriteList"""
+        return RawTableWriteList([table.as_write() for table in self.data])
+
+    def as_ids(self) -> list[RawTableID]:
+        """Returns this TableList as a list of TableIDs"""
+        return [table.as_id() for table in self.data]
+
+
+T_ID = TypeVar("T_ID", bound=Hashable)
 
 
 # Inheriting from dict as we are extending it,

cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py

@@ -3,6 +3,7 @@ from graphlib import TopologicalSorter
 from typing import Any, Literal, cast
 
 from cognite.client import CogniteClient
+from cognite.client.data_classes import filters
 from cognite.client.data_classes._base import (
     T_CogniteResourceList,
     T_WritableCogniteResource,
@@ -95,6 +96,42 @@ class SpaceLoader(DataModelingLoader[str, SpaceApply, Space, SpaceApplyList, Spa
             ids = [cast(Space | SpaceApply, item).space for item in ids]
         return self.client.data_modeling.spaces.delete(cast(SequenceNotStr[str], ids))
 
+    def clean(self, space: str) -> None:
+        """Deletes all data in a space.
+
+        This means all nodes, edges, views, containers, and data models located in the given space.
+
+        Args:
+            client: Connected CogniteClient
+            space: The space to delete.
+
+        """
+        edges = self.client.data_modeling.instances.list(
+            "edge", limit=-1, filter=filters.Equals(["edge", "space"], space)
+        )
+        if edges:
+            instances = self.client.data_modeling.instances.delete(edges=edges.as_ids())
+            print(f"Deleted {len(instances.edges)} edges")
+        nodes = self.client.data_modeling.instances.list(
+            "node", limit=-1, filter=filters.Equals(["node", "space"], space)
+        )
+        if nodes:
+            instances = self.client.data_modeling.instances.delete(nodes=nodes.as_ids())
+            print(f"Deleted {len(instances.nodes)} nodes")
+        views = self.client.data_modeling.views.list(limit=-1, space=space)
+        if views:
+            deleted_views = self.client.data_modeling.views.delete(views.as_ids())
+            print(f"Deleted {len(deleted_views)} views")
+        containers = self.client.data_modeling.containers.list(limit=-1, space=space)
+        if containers:
+            deleted_containers = self.client.data_modeling.containers.delete(containers.as_ids())
+            print(f"Deleted {len(deleted_containers)} containers")
+        if data_models := self.client.data_modeling.data_models.list(limit=-1, space=space):
+            deleted_data_models = self.client.data_modeling.data_models.delete(data_models.as_ids())
+            print(f"Deleted {len(deleted_data_models)} data models")
+        deleted_space = self.client.data_modeling.spaces.delete(space)
+        print(f"Deleted space {deleted_space}")
+
 
 class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, ViewList]):
     resource_name = "views"

cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py

@@ -15,8 +15,9 @@ from cognite.client.data_classes import (
 from cognite.client.exceptions import CogniteAPIError
 from cognite.client.utils.useful_types import SequenceNotStr
 
+from cognite.neat.utils.cdf.data_classes import RawTable, RawTableID, RawTableList, RawTableWrite, RawTableWriteList
+
 from ._base import ResourceLoader
-from .data_classes import RawTable, RawTableID, RawTableList, RawTableWrite, RawTableWriteList
 
 
 class TransformationLoader(

cognite/neat/utils/collection_.py

@@ -0,0 +1,18 @@
+from collections import Counter
+from collections.abc import Sequence
+from typing import TypeVar
+
+T_Element = TypeVar("T_Element")
+
+
+def remove_none_elements_from_set(s: set[T_Element]) -> set[T_Element]:
+    return {x for x in s if x is not None}
+
+
+def most_occurring_element(list_of_elements: list[T_Element]) -> T_Element:
+    counts = Counter(list_of_elements)
+    return counts.most_common(1)[0][0]
+
+
+def chunker(sequence: Sequence[T_Element], chunk_size: int) -> list[Sequence[T_Element]]:
+    return [sequence[i : i + chunk_size] for i in range(0, len(sequence), chunk_size)]

cognite/neat/utils/rdf_.py

@@ -0,0 +1,165 @@
+import re
+from typing import Any, Literal, TypeAlias, cast, overload
+
+from pydantic import HttpUrl, TypeAdapter, ValidationError
+from rdflib import Literal as RdfLiteral
+from rdflib import Namespace, URIRef
+
+Triple: TypeAlias = tuple[URIRef, URIRef, RdfLiteral | URIRef]
+
+
+@overload
+def remove_namespace_from_uri(
+    *URI: URIRef | str,
+    special_separator: str = "#_",
+    validation: Literal["full", "prefix"] = "prefix",
+) -> str: ...
+
+
+@overload
+def remove_namespace_from_uri(
+    *URI: tuple[URIRef | str, ...],
+    special_separator: str = "#_",
+    validation: Literal["full", "prefix"] = "prefix",
+) -> tuple[str, ...]: ...
+
+
+def remove_namespace_from_uri(
+    *URI: URIRef | str | tuple[URIRef | str, ...],
+    special_separator: str = "#_",
+    validation: Literal["full", "prefix"] = "prefix",
+) -> tuple[str, ...] | str:
+    """Removes namespace from URI
+
+    Args
+        URI: URIRef | str
+            URI of an entity
+        special_separator : str
+            Special separator to use instead of # or / if present in URI
+            Set by default to "#_" which covers special client use case
+        validation: str
+            Validation type to use for URI. If set to "full", URI will be validated using pydantic
+            If set to "prefix", only check if URI starts with http or https will be made
+
+    Returns
+        Entities id without namespace
+
+    Examples:
+
+        >>> remove_namespace_from_uri("http://www.example.org/index.html#section2")
+        'section2'
+        >>> remove_namespace_from_uri("http://www.example.org/index.html#section2", "http://www.example.org/index.html#section3")
+        ('section2', 'section3')
+    """
+    if isinstance(URI, str | URIRef):
+        uris = (URI,)
+    elif isinstance(URI, tuple):
+        # Assume that all elements in the tuple are of the same type following type hint
+        uris = cast(tuple[URIRef | str, ...], URI)
+    else:
+        raise TypeError(f"URI must be of type URIRef or str, got {type(URI)}")
+
+    output = []
+    for u in uris:
+        if validation == "full":
+            try:
+                _ = TypeAdapter(HttpUrl).validate_python(u)
+                output.append(u.split(special_separator if special_separator in u else "#" if "#" in u else "/")[-1])
+            except ValidationError:
+                output.append(str(u))
+        else:
+            if u.lower().startswith("http"):
+                output.append(u.split(special_separator if special_separator in u else "#" if "#" in u else "/")[-1])
+            else:
+                output.append(str(u))
+
+    return tuple(output) if len(output) > 1 else output[0]
+
+
+def get_namespace(URI: URIRef, special_separator: str = "#_") -> str:
+    """Removes namespace from URI
+
+    Parameters
+    ----------
+    URI : URIRef
+        URI of an entity
+    special_separator : str
+        Special separator to use instead of # or / if present in URI
+        Set by default to "#_" which covers special client use case
+
+    Returns
+    -------
+    str
+        Entity id without namespace
+    """
+    if special_separator in URI:
+        return URI.split(special_separator)[0] + special_separator
+    elif "#" in URI:
+        return URI.split("#")[0] + "#"
+    else:
+        return "/".join(URI.split("/")[:-1]) + "/"
+
+
+def as_neat_compliant_uri(uri: URIRef) -> URIRef:
+    namespace = get_namespace(uri)
+    id_ = remove_namespace_from_uri(uri)
+    compliant_uri = re.sub(r"[^a-zA-Z0-9-_.]", "", id_)
+    return URIRef(f"{namespace}{compliant_uri}")
+
+
+def convert_rdflib_content(content: RdfLiteral | URIRef | dict | list) -> Any:
+    if isinstance(content, RdfLiteral) or isinstance(content, URIRef):
+        return content.toPython()
+    elif isinstance(content, dict):
+        return {key: convert_rdflib_content(value) for key, value in content.items()}
+    elif isinstance(content, list):
+        return [convert_rdflib_content(item) for item in content]
+    else:
+        return content
+
+
+def uri_to_short_form(URI: URIRef, prefixes: dict[str, Namespace]) -> str | URIRef:
+    """Returns the short form of a URI if its namespace is present in the prefixes dict,
+    otherwise returns the URI itself
+
+    Args:
+        URI: URI to be converted to form prefix:entityName
+        prefixes: dict of prefixes
+
+    Returns:
+        shortest form of the URI if its namespace is present in the prefixes dict,
+        otherwise returns the URI itself
+    """
+    uris: set[str | URIRef] = {URI}
+    for prefix, namespace in prefixes.items():
+        if URI.startswith(namespace):
+            uris.add(f"{prefix}:{URI.replace(namespace, '')}")
+    return min(uris, key=len)
+
+
+def _traverse(hierarchy: dict, graph: dict, names: list[str]) -> dict:
+    """traverse the graph and return the hierarchy"""
+    for name in names:
+        hierarchy[name] = _traverse({}, graph, graph[name])
+    return hierarchy
+
+
+def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list:
+    """Returns the inheritance path for a given child
+
+    Args:
+        child: Child class
+        child_parent: Dictionary of child to parent relationship
+
+    Returns:
+        Inheritance path for a given child
+
+    !!! note "No Circular Inheritance"
+        This method assumes that the child_parent dictionary is a tree and does not contain any cycles.
+    """
+    path = []
+    if child in child_parent:
+        path.extend(child_parent[child])
+        for parent in child_parent[child]:
+            path.extend(get_inheritance_path(parent, child_parent))
+    return path

cognite/neat/utils/text.py

@@ -102,3 +102,7 @@ def to_snake(string: str) -> str:
     else:
         words = pattern.findall(string)
     return "_".join(map(str.lower, words))
+
+
+def replace_non_alphanumeric_with_underscore(text: str) -> str:
+    return re.sub(r"\W+", "_", text)

cognite/neat/utils/time_.py

@@ -0,0 +1,17 @@
+import sys
+from datetime import datetime
+
+if sys.version_info >= (3, 11):
+    from datetime import UTC
+else:
+    from datetime import timezone
+
+    UTC = timezone.utc
+
+
+def epoch_now_ms() -> int:
+    return int((datetime.now(UTC) - datetime(1970, 1, 1, tzinfo=UTC)).total_seconds() * 1000)
+
+
+def datetime_utc_now():
+    return datetime.now(UTC)

cognite/neat/utils/upload.py

@@ -36,6 +36,7 @@ class UploadResultList(NeatList[UploadResultCore]): ...
 @dataclass
 class UploadResult(UploadResultCore, Generic[T_ID]):
     created: set[T_ID] = field(default_factory=set)
+    upserted: set[T_ID] = field(default_factory=set)
     deleted: set[T_ID] = field(default_factory=set)
     changed: set[T_ID] = field(default_factory=set)
     unchanged: set[T_ID] = field(default_factory=set)
@@ -53,12 +54,21 @@ class UploadResult(UploadResultCore, Generic[T_ID]):
 
     @property
     def success(self) -> int:
-        return len(self.created) + len(self.deleted) + len(self.changed) + len(self.unchanged) + len(self.skipped)
+        return (
+            len(self.created)
+            + len(self.deleted)
+            + len(self.changed)
+            + len(self.upserted)
+            + len(self.unchanged)
+            + len(self.skipped)
+        )
 
     def dump(self, aggregate: bool = True) -> dict[str, Any]:
         output = super().dump(aggregate)
         if self.created:
             output["created"] = len(self.created) if aggregate else list(self.created)
+        if self.upserted:
+            output["upserted"] = len(self.upserted) if aggregate else list(self.upserted)
         if self.deleted:
             output["deleted"] = len(self.deleted) if aggregate else list(self.deleted)
         if self.changed:
@@ -69,6 +79,8 @@ class UploadResult(UploadResultCore, Generic[T_ID]):
             output["skipped"] = len(self.skipped) if aggregate else list(self.skipped)
         if self.failed_created:
             output["failed_created"] = len(self.failed_created) if aggregate else list(self.failed_created)
+        if self.failed_upserted:
+            output["failed_upserted"] = len(self.failed_upserted) if aggregate else list(self.failed_upserted)
         if self.failed_changed:
             output["failed_changed"] = len(self.failed_changed) if aggregate else list(self.failed_changed)
         if self.failed_deleted:

cognite/neat/workflows/_exceptions.py

@@ -1,7 +1,7 @@
-from cognite.neat.utils.exceptions import NeatError
+from cognite.neat.exceptions import NeatException
 
 
-class InvalidStepOutputException(NeatError):
+class InvalidStepOutputException(NeatException):
     type_ = "invalid_step_output"
     code = 400
     description = "The step output is invalid."
@@ -12,7 +12,7 @@ class InvalidStepOutputException(NeatError):
         super().__init__(self.message)
 
 
-class ConfigurationNotSet(NeatError):
+class ConfigurationNotSet(NeatException):
     type_ = "configuration_not_set"
     code = 400
     description = "The configuration is not set."
@@ -29,13 +29,13 @@ class ConfigurationNotSet(NeatError):
         return self.message
 
 
-class StepNotInitialized(NeatError):
+class StepNotInitialized(NeatException):
     def __init__(self, step_name: str):
         self.message = f"Step {step_name} has not been initialized."
         super().__init__(self.message)
 
 
-class StepFlowContextNotInitialized(NeatError):
+class StepFlowContextNotInitialized(NeatException):
     def __init__(self, step_name: str):
         self.message = f"Step {step_name} requires flow context which is missing."
         super().__init__(self.message)

cognite/neat/workflows/base.py

@@ -13,7 +13,7 @@ from prometheus_client import Gauge
 
 from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
 from cognite.neat.config import Config
-from cognite.neat.utils.utils import retry_decorator
+from cognite.neat.utils.auxiliary import retry_decorator
 from cognite.neat.workflows import cdf_store, utils
 from cognite.neat.workflows._exceptions import ConfigurationNotSet, InvalidStepOutputException
 from cognite.neat.workflows.cdf_store import CdfStore

cognite/neat/workflows/steps/lib/current/rules_validator.py

@@ -9,7 +9,7 @@ from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.issues.dms import MissingContainerError, MissingSpaceError, MissingViewError
 from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
 from cognite.neat.rules.models import DMSRules, SchemaCompleteness
-from cognite.neat.utils import cdf_loaders
+from cognite.neat.utils.cdf.loaders import ViewLoader
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import MultiRuleData
@@ -69,7 +69,7 @@ class ValidateRulesAgainstCDF(Step):
         retrieved_containers = cdf_client.data_modeling.containers.retrieve(missing_containers).as_write()
         # Converting read format of views to write format requires to account for parents (implements)
         # Thus we use the loader to convert the views to write format.
-        view_loader = cdf_loaders.ViewLoader(cdf_client)
+        view_loader = ViewLoader(cdf_client)
         retrieved_views = [
             view_loader.as_write(view) for view in cdf_client.data_modeling.views.retrieve(missing_views)
         ]

cognite/neat/workflows/steps/lib/legacy/graph_extractor.py

@@ -14,7 +14,7 @@ from cognite.neat.legacy.graph.extractors._mock_graph_generator import (
     generate_triples as generate_mock_triples,
 )
 from cognite.neat.legacy.rules.exporters._rules2triples import get_instances_as_triples
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import (

cognite/neat/workflows/steps/lib/legacy/graph_loader.py

@@ -26,7 +26,7 @@ from cognite.neat.legacy.graph.loaders.core.rdf_to_relationships import (
 from cognite.neat.legacy.graph.loaders.rdf_to_dms import upload_edges, upload_nodes
 from cognite.neat.legacy.graph.loaders.validator import validate_asset_hierarchy
 from cognite.neat.legacy.rules.models.rdfpath import TransformationRuleType
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.auxiliary import generate_exception_report
 from cognite.neat.workflows._exceptions import StepFlowContextNotInitialized, StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import (

cognite/neat/workflows/steps/lib/legacy/rules_exporter.py

@@ -12,7 +12,7 @@ from cognite.neat.legacy.rules import exporters
 from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
 from cognite.neat.legacy.rules.exporters._rules2graphql import GraphQLSchema
 from cognite.neat.legacy.rules.exporters._rules2ontology import Ontology
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.auxiliary import generate_exception_report
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import CogniteClient, DMSSchemaComponentsData, RulesData

cognite/neat/workflows/steps/lib/legacy/rules_importer.py

@@ -13,7 +13,7 @@ from cognite.neat.legacy.rules import exporters, importers
 from cognite.neat.legacy.rules.models.rdfpath import TransformationRuleType
 from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
 from cognite.neat.legacy.rules.models.value_types import ValueType
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.auxiliary import generate_exception_report
 from cognite.neat.workflows import utils
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.cdf_store import CdfStore

{cognite_neat-0.87.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.87.0
+Version: 0.87.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -21,7 +21,7 @@ Provides-Extra: oxi
 Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
-Requires-Dist: cognite-sdk (>=7.52.0,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.54.6,<8.0.0)
 Requires-Dist: deepdiff
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
 Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"