cognite-neat 0.75.6__tar.gz → 0.75.7__tar.gz

{cognite_neat-0.75.6 → cognite_neat-0.75.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.75.6
+Version: 0.75.7
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.75.7/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.75.7"

{cognite_neat-0.75.6 → cognite_neat-0.75.7}/cognite/neat/rules/exporters/_models.py

@@ -30,6 +30,7 @@ class UploadResult(UploadResultCore):
     skipped: int = 0
     failed_created: int = 0
     failed_changed: int = 0
+    failed_deleted: int = 0
     error_messages: list[str] = field(default_factory=list)
 
     @property
@@ -38,7 +39,7 @@ class UploadResult(UploadResultCore):
 
     @property
     def failed(self) -> int:
-        return self.failed_created + self.failed_changed
+        return self.failed_created + self.failed_changed + self.failed_deleted
 
     def as_report_str(self) -> str:
         line = []
@@ -50,9 +51,13 @@ class UploadResult(UploadResultCore):
             line.append(f"skipped {self.skipped}")
         if self.unchanged:
             line.append(f"unchanged {self.unchanged}")
+        if self.deleted:
+            line.append(f"deleted {self.deleted}")
         if self.failed_created:
             line.append(f"failed to create {self.failed_created}")
         if self.failed_changed:
             line.append(f"failed to update {self.failed_changed}")
+        if self.failed_deleted:
+            line.append(f"failed to delete {self.failed_deleted}")
 
         return f"{self.name.title()}: {', '.join(line)}"

{cognite_neat-0.75.6 → cognite_neat-0.75.7}/cognite/neat/rules/exporters/_rules2dms.py

@@ -151,21 +151,55 @@ class DMSExporter(CDFExporter[DMSSchema]):
             schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
         return schema
 
+    def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
+        schema, to_export = self._prepare_schema_and_exporters(rules, client)
+
+        # we need to reverse order in which we are picking up the items to delete
+        # as they are sorted in the order of creation and we need to delete them in reverse order
+        for all_items, loader in reversed(to_export):
+            all_item_ids = loader.get_ids(all_items)
+            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
+            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
+            cdf_items = loader.retrieve(item_ids)
+            cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
+            items = [item for item in all_items if loader.get_id(item) in item_ids]
+            to_delete = []
+
+            for item in items:
+                if (
+                    isinstance(loader, DataModelingLoader)
+                    and self.include_space is not None
+                    and not loader.in_space(item, self.include_space)
+                ):
+                    continue
+
+                cdf_item = cdf_item_by_id.get(loader.get_id(item))
+                if cdf_item:
+                    to_delete.append(cdf_item)
+
+            deleted = len(to_delete)
+            failed_deleted = 0
+
+            error_messages: list[str] = []
+            if not dry_run:
+                if to_delete:
+                    try:
+                        loader.delete(to_delete)
+                    except CogniteAPIError as e:
+                        failed_deleted = len(e.failed) + len(e.unknown)
+                        deleted -= failed_deleted
+                        error_messages.append(f"Failed delete: {e.message}")
+
+            yield UploadResult(
+                name=loader.resource_name,
+                deleted=deleted,
+                skipped=skipped,
+                failed_deleted=failed_deleted,
+                error_messages=error_messages,
+            )
+
     def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
-        schema = self.export(rules)
-        to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
-        if self.export_components.intersection({"all", "spaces"}):
-            to_export.append((schema.spaces, SpaceLoader(client)))
-        if self.export_components.intersection({"all", "containers"}):
-            to_export.append((schema.containers, ContainerLoader(client)))
-        if self.export_components.intersection({"all", "views"}):
-            to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
-        if self.export_components.intersection({"all", "data_models"}):
-            to_export.append((schema.data_models, DataModelLoader(client)))
-        if isinstance(schema, PipelineSchema):
-            to_export.append((schema.databases, RawDatabaseLoader(client)))
-            to_export.append((schema.raw_tables, RawTableLoader(client)))
-            to_export.append((schema.transformations, TransformationLoader(client)))
+        schema, to_export = self._prepare_schema_and_exporters(rules, client)
 
         # The conversion from DMS to GraphQL does not seem to be triggered even if the views
         # are changed. This is a workaround to force the conversion.
@@ -254,3 +288,22 @@ class DMSExporter(CDFExporter[DMSSchema]):
 
             if loader.resource_name == "views" and (created or changed) and not redeploy_data_model:
                 redeploy_data_model = True
+
+    def _prepare_schema_and_exporters(
+        self, rules, client
+    ) -> tuple[DMSSchema, list[tuple[CogniteResourceList, ResourceLoader]]]:
+        schema = self.export(rules)
+        to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
+        if self.export_components.intersection({"all", "spaces"}):
+            to_export.append((schema.spaces, SpaceLoader(client)))
+        if self.export_components.intersection({"all", "containers"}):
+            to_export.append((schema.containers, ContainerLoader(client)))
+        if self.export_components.intersection({"all", "views"}):
+            to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
+        if self.export_components.intersection({"all", "data_models"}):
+            to_export.append((schema.data_models, DataModelLoader(client)))
+        if isinstance(schema, PipelineSchema):
+            to_export.append((schema.databases, RawDatabaseLoader(client)))
+            to_export.append((schema.raw_tables, RawTableLoader(client)))
+            to_export.append((schema.transformations, TransformationLoader(client)))
+        return schema, to_export

cognite_neat-0.75.7/cognite/neat/rules/models/data_types.py

@@ -0,0 +1,259 @@
+import sys
+from datetime import date, datetime
+from typing import Any, ClassVar
+
+from cognite.client.data_classes import data_modeling as dms
+from pydantic import BaseModel, model_serializer, model_validator
+from pydantic.functional_validators import ModelWrapValidatorHandler
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class DataType(BaseModel):
+    name: ClassVar[str]
+    python: ClassVar[type]
+    dms: ClassVar[type[dms.PropertyType]]
+    graphql: ClassVar[str]
+    xsd: ClassVar[str]
+    sql: ClassVar[str]
+
+    @classmethod
+    def load(cls, data: Any) -> Self:
+        return cls.model_validate(data)
+
+    def dump(self) -> dict[str, Any]:
+        return self.model_dump(by_alias=True)
+
+    @model_validator(mode="wrap")
+    def _load(cls, value: Any, handler: ModelWrapValidatorHandler["DataType"]) -> Any:
+        if isinstance(value, cls | dict):
+            return value
+        elif isinstance(value, str):
+            try:
+                return _DATA_TYPE_BY_NAME[value.casefold()]()
+            except KeyError:
+                raise ValueError(f"Unknown literal type: {value}") from None
+        raise ValueError(f"Cannot load {cls.__name__} from {value}")
+
+    @model_serializer(when_used="unless-none", return_type=str)
+    def as_str(self) -> str:
+        return str(self)
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, type(self))
+
+
+class Boolean(DataType):
+    name = "boolean"
+    python = bool
+    dms = dms.Boolean
+    graphql = "Boolean"
+    xsd = "xsd:boolean"
+    sql = "BOOLEAN"
+
+
+class Float(DataType):
+    name = "float"
+    python = float
+    dms = dms.Float32
+    graphql = "Float"
+    xsd = "xsd:float"
+    sql = "FLOAT"
+
+
+class Double(DataType):
+    name = "double"
+    python = float
+    dms = dms.Float64
+    graphql = "Float"
+    xsd = "xsd:double"
+    sql = "FLOAT"
+
+
+class Integer(DataType):
+    name = "integer"
+    python = int
+    dms = dms.Int32
+    graphql = "Int"
+    xsd = "xsd:integer"
+    sql = "INTEGER"
+
+
+class NonPositiveInteger(DataType):
+    name = "nonPositiveInteger"
+    python = int
+    dms = dms.Int32
+    graphql = "Int"
+    xsd = "xsd:nonPositiveInteger"
+    sql = "INTEGER"
+
+
+class NonNegativeInteger(DataType):
+    name = "nonNegativeInteger"
+    python = int
+    dms = dms.Int32
+    graphql = "Int"
+    xsd = "xsd:nonNegativeInteger"
+    sql = "INTEGER"
+
+
+class NegativeInteger(DataType):
+    name = "negativeInteger"
+    python = int
+    dms = dms.Int32
+    graphql = "Int"
+    xsd = "xsd:negativeInteger"
+    sql = "INTEGER"
+
+
+class Long(DataType):
+    name = "long"
+    python = int
+    dms = dms.Int64
+    graphql = "Int"
+    xsd = "xsd:long"
+    sql = "BIGINT"
+
+
+class String(DataType):
+    name = "string"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class LangString(DataType):
+    name = "langString"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class AnyURI(DataType):
+    name = "anyURI"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:anyURI"
+    sql = "STRING"
+
+
+class NormalizedString(DataType):
+    name = "normalizedString"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:normalizedString"
+    sql = "STRING"
+
+
+class Token(DataType):
+    name = "token"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class DateTime(DataType):
+    name = "dateTime"
+    python = datetime
+    dms = dms.Timestamp
+    graphql = "Timestamp"
+    xsd = "xsd:dateTimeStamp"
+    sql = "TIMESTAMP"
+
+
+class DateTimeStamp(DataType):
+    name = "dateTimeStamp"
+    python = datetime
+    dms = dms.Timestamp
+    graphql = "Timestamp"
+    xsd = "xsd:dateTimeStamp"
+    sql = "TIMESTAMP"
+
+
+class Timestamp(DataType):
+    name = "timestamp"
+    python = datetime
+    dms = dms.Timestamp
+    graphql = "Timestamp"
+    xsd = "xsd:dateTimeStamp"
+    sql = "TIMESTAMP"
+
+
+class Date(DataType):
+    name = "date"
+    python = date
+    dms = dms.Date
+    graphql = "String"
+    xsd = "xsd:date"
+    sql = "DATE"
+
+
+class PlainLiteral(DataType):
+    name = "PlainLiteral"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class Literal(DataType):
+    name = "Literal"
+    python = str
+    dms = dms.Text
+    graphql = "String"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class Timeseries(DataType):
+    name = "timeseries"
+    python = dms.TimeSeriesReference
+    dms = dms.TimeSeriesReference
+    graphql = "TimeSeries"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class File(DataType):
+    name = "file"
+    python = dms.FileReference
+    dms = dms.FileReference
+    graphql = "File"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class Sequence(DataType):
+    name = "sequence"
+    python = dms.SequenceReference
+    dms = dms.SequenceReference
+    graphql = "Sequence"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+class Json(DataType):
+    name = "json"
+    python = dms.Json
+    dms = dms.Json
+    graphql = "Json"
+    xsd = "xsd:string"
+    sql = "STRING"
+
+
+_DATA_TYPE_BY_NAME = {cls.name.casefold(): cls for cls in DataType.__subclasses__()}

cognite_neat-0.75.7/cognite/neat/rules/models/entities.py

@@ -0,0 +1,299 @@
+import re
+import sys
+import threading
+from abc import ABC, abstractmethod
+from functools import total_ordering
+from typing import Annotated, Any, ClassVar, Generic, TypeVar
+
+from cognite.client.data_classes.data_modeling.ids import ContainerId, DataModelId, PropertyId, ViewId
+from pydantic import BaseModel, BeforeValidator, Field, PlainSerializer, model_serializer, model_validator
+
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+    from typing import Self
+else:
+    from backports.strenum import StrEnum
+    from typing_extensions import Self
+
+
+class EntityTypes(StrEnum):
+    view_non_versioned = "view_non_versioned"
+    subject = "subject"
+    predicate = "predicate"
+    object = "object"
+    class_ = "class"
+    parent_class = "parent_class"
+    property_ = "property"
+    object_property = "ObjectProperty"
+    data_property = "DatatypeProperty"
+    annotation_property = "AnnotationProperty"
+    object_value_type = "object_value_type"
+    data_value_type = "data_value_type"  # these are strings, floats, ...
+    xsd_value_type = "xsd_value_type"
+    dms_value_type = "dms_value_type"
+    view = "view"
+    reference_entity = "reference_entity"
+    container = "container"
+    datamodel = "datamodel"
+    undefined = "undefined"
+
+
+# ALLOWED
+_ALLOWED_PATTERN = r"[^a-zA-Z0-9-_.]"
+
+# FOR PARSING STRINGS:
+_PREFIX_REGEX = r"[a-zA-Z]+[a-zA-Z0-9-_.]*[a-zA-Z0-9]+"
+_SUFFIX_REGEX = r"[a-zA-Z0-9-_.]+[a-zA-Z0-9]|[-_.]*[a-zA-Z0-9]+"
+_VERSION_REGEX = r"[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?"
+_PROPERTY_REGEX = r"[a-zA-Z0-9][a-zA-Z0-9_-]*[a-zA-Z0-9]?"
+_ENTITY_ID_REGEX = rf"{_PREFIX_REGEX}:({_SUFFIX_REGEX})"
+_ENTITY_ID_REGEX_COMPILED = re.compile(rf"^(?P<prefix>{_PREFIX_REGEX}):(?P<suffix>{_SUFFIX_REGEX})$")
+_VERSIONED_ENTITY_REGEX_COMPILED = re.compile(
+    rf"^(?P<prefix>{_PREFIX_REGEX}):(?P<suffix>{_SUFFIX_REGEX})\(version=(?P<version>{_VERSION_REGEX})\)$"
+)
+_CLASS_ID_REGEX = rf"(?P<{EntityTypes.class_}>{_ENTITY_ID_REGEX})"
+_CLASS_ID_REGEX_COMPILED = re.compile(rf"^{_CLASS_ID_REGEX}$")
+_PROPERTY_ID_REGEX = rf"\((?P<{EntityTypes.property_}>{_ENTITY_ID_REGEX})\)"
+
+_ENTITY_PATTERN = re.compile(r"^(?P<prefix>.*?):?(?P<suffix>[^(:]*)(\((?P<content>[^)]+)\))?$")
+
+
+class _Undefined(BaseModel):
+    ...
+
+
+class _Unknown(BaseModel):
+    def __str__(self) -> str:
+        return "#N/A"
+
+
+# This is a trick to make Undefined and Unknown singletons
+Undefined = _Undefined()
+Unknown = _Unknown()
+
+
+@total_ordering
+class Entity(BaseModel):
+    """Entity is a class or property in OWL/RDF sense."""
+
+    type_: ClassVar[EntityTypes] = EntityTypes.undefined
+    prefix: str | _Undefined = Undefined
+    suffix: str | _Unknown
+
+    @classmethod
+    def load(cls, data: Any) -> Self:
+        return cls.model_validate(data)
+
+    @model_validator(mode="before")
+    def _load(cls, data: Any) -> dict:
+        if isinstance(data, cls):
+            return data.model_dump()
+        elif isinstance(data, dict):
+            return data
+        elif hasattr(data, "versioned_id"):
+            # Todo: Remove. Is here for backwards compatibility
+            data = data.versioned_id
+        elif not isinstance(data, str):
+            raise ValueError(f"Cannot load {cls.__name__} from {data}")
+
+        return cls._parse(data)
+
+    @model_serializer(when_used="unless-none", return_type=str)
+    def as_str(self) -> str:
+        return str(self)
+
+    @classmethod
+    def _parse(cls, raw: str) -> dict:
+        if not (result := _ENTITY_PATTERN.match(raw)):
+            return dict(prefix=Undefined, suffix=Unknown)
+        prefix = result.group("prefix") or Undefined
+        suffix = result.group("suffix")
+        content = result.group("content")
+        if content is None:
+            return dict(prefix=prefix, suffix=suffix)
+        extra_args = dict(pair.strip().split("=") for pair in content.split(","))
+        expected_args = {field_.alias or field_name for field_name, field_ in cls.model_fields.items()}
+        for key in list(extra_args):
+            if key not in expected_args:
+                # Todo Warning about unknown key
+                del extra_args[key]
+        return dict(prefix=prefix, suffix=suffix, **extra_args)
+
+    def dump(self) -> str:
+        return str(self)
+
+    def as_tuple(self) -> tuple[str, ...]:
+        # We haver overwritten the serialization to str, so we need to do it manually
+        extra: tuple[str, ...] = tuple(
+            [
+                str(v or "")
+                for field_name in self.model_fields
+                if isinstance(v := getattr(self, field_name), str | None) and field_name not in {"prefix", "suffix"}
+            ]
+        )
+        if isinstance(self.prefix, _Undefined):
+            return str(self.suffix), *extra
+        else:
+            return self.prefix, str(self.suffix), *extra
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, Entity):
+            return NotImplemented
+        return self.as_tuple() < other.as_tuple()
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Entity):
+            return NotImplemented
+        return self.as_tuple() == other.as_tuple()
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
+    def __str__(self) -> str:
+        return self.id
+
+    def __repr__(self) -> str:
+        # We have overwritten the serialization to str, so we need to do it manually
+        model_dump = ((k, v) for k in self.model_fields if (v := getattr(self, k)) is not None)
+        args = ",".join([f"{k}={v}" for k, v in model_dump])
+        return f"{self.type_.value}({args})"
+
+    @property
+    def id(self) -> str:
+        # We have overwritten the serialization to str, so we need to do it manually
+        model_dump = (
+            (field.alias or field_name, v)
+            for field_name, field in self.model_fields.items()
+            if (v := getattr(self, field_name)) is not None and field_name not in {"prefix", "suffix"}
+        )
+        args = ",".join([f"{k}={v}" for k, v in model_dump])
+        if self.prefix is Undefined:
+            base_id = str(self.suffix)
+        else:
+            base_id = f"{self.prefix}:{self.suffix!s}"
+        if args:
+            return f"{base_id}({args})"
+        else:
+            return base_id
+
+    @property
+    def versioned_id(self) -> str:
+        # Todo: Remove. Is here for backwards compatibility
+        return self.id
+
+    def as_non_versioned_entity(self) -> str:
+        # Todo: Remove. Is here for backwards compatibility
+        if self.prefix is Undefined:
+            return f"{self.suffix!s}"
+        return f"{self.prefix}:{self.suffix!s}"
+
+
+class ClassEntity(Entity):
+    type_: ClassVar[EntityTypes] = EntityTypes.class_
+    version: str | None = None
+
+
+class ParentClassEntity(ClassEntity):
+    type_: ClassVar[EntityTypes] = EntityTypes.parent_class
+
+    def as_class_entity(self) -> ClassEntity:
+        return ClassEntity(prefix=self.prefix, suffix=self.suffix, version=self.version)
+
+
+T_ID = TypeVar("T_ID", bound=ContainerId | ViewId | DataModelId | PropertyId)
+
+
+class DMSEntity(Entity, Generic[T_ID], ABC):
+    type_: ClassVar[EntityTypes] = EntityTypes.undefined
+    default_space_by_thread: ClassVar[dict[threading.Thread, str]] = {}
+    suffix: str
+
+    @classmethod
+    def set_default_space(cls, space: str) -> None:
+        cls.default_space_by_thread[threading.current_thread()] = space
+
+    @property
+    def space(self) -> str:
+        """Returns entity space in CDF."""
+        if isinstance(self.prefix, _Undefined):
+            return self.default_space_by_thread.get(threading.current_thread(), "MISSING")
+        else:
+            return self.prefix
+
+    @property
+    def external_id(self) -> str:
+        """Returns entity external id in CDF."""
+        return self.suffix
+
+    @abstractmethod
+    def as_id(self) -> T_ID:
+        raise NotImplementedError("Method as_id must be implemented in subclasses")
+
+
+class ContainerEntity(DMSEntity[ContainerId]):
+    type_: ClassVar[EntityTypes] = EntityTypes.container
+
+    def as_id(self) -> ContainerId:
+        return ContainerId(space=self.space, external_id=self.external_id)
+
+
+class DMSVersionedEntity(DMSEntity[T_ID], ABC):
+    version: str | None = None
+    default_version_by_thread: ClassVar[dict[threading.Thread, str]] = {}
+
+    @property
+    def version_with_fallback(self) -> str:
+        if self.version is not None:
+            return self.version
+        return self.default_version_by_thread.get(threading.current_thread(), "MISSING")
+
+
+class ViewEntity(DMSVersionedEntity[ViewId]):
+    type_: ClassVar[EntityTypes] = EntityTypes.view
+
+    def as_id(
+        self,
+    ) -> ViewId:
+        return ViewId(space=self.space, external_id=self.external_id, version=self.version_with_fallback)
+
+
+class PropertyEntity(DMSVersionedEntity[PropertyId]):
+    type_: ClassVar[EntityTypes] = EntityTypes.property_
+    property_: str = Field(alias="property")
+
+    def as_id(self) -> PropertyId:
+        return PropertyId(
+            source=ViewId(self.space, self.external_id, self.version_with_fallback), property=self.property_
+        )
+
+
+class DataModelEntity(DMSVersionedEntity[DataModelId]):
+    type_: ClassVar[EntityTypes] = EntityTypes.datamodel
+
+    def as_id(self) -> DataModelId:
+        return DataModelId(space=self.space, external_id=self.external_id, version=self.version_with_fallback)
+
+
+class ReferenceEntity(PropertyEntity):
+    type_: ClassVar[EntityTypes] = EntityTypes.reference_entity
+
+
+def _split_str(v: Any) -> list[str]:
+    if isinstance(v, str):
+        return v.replace(", ", ",").split(",")
+    return v
+
+
+def _join_str(v: list[ClassEntity]) -> str | None:
+    return ",".join([entry.id for entry in v]) if v else None
+
+
+ParentEntityList = Annotated[
+    list[ParentClassEntity],
+    BeforeValidator(_split_str),
+    PlainSerializer(
+        _join_str,
+        return_type=str,
+        when_used="unless-none",
+    ),
+]