cognite-neat 0.123.26__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_data_model/importers/_table_importer/source.py

@@ -0,0 +1,94 @@
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+
+from .data_classes import DMS_API_MAPPING
+
+
+@dataclass
+class SpreadsheetReadContext:
+    """This class is used to store information about the source spreadsheet.
+
+    It is used to adjust row numbers to account for header rows and empty rows
+    such that the error/warning messages are accurate.
+    """
+
+    header_row: int = 0
+    empty_rows: list[int] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        self.empty_rows.sort()
+
+    def adjusted_row_number(self, row_no: int) -> int:
+        """Adjusts the given row number to account for header rows and empty rows.
+
+        Args:
+            row_no (int): The original row number (0-based).
+
+        !!! note "Row Numbering"
+            Input rows are zero-indexed, while output rows are one-indexed as they appear in Excel.
+            Therefore, we are adding 1 to offset header and 1 to offset row no
+
+        """
+        output = (row_no + 1) + (self.header_row + 1)
+        counter = 0
+        for empty_row in self.empty_rows:
+            if empty_row < self.header_row:
+                continue
+            if empty_row <= output:
+                counter += 1
+                output += 1
+            else:
+                break
+
+        return output
+
+
+@dataclass
+class TableSource:
+    source: str
+    table_read: dict[str, SpreadsheetReadContext] = field(default_factory=dict)
+
+    def location(self, path: tuple[int | str, ...]) -> str:
+        table_id: str | None = None
+        row_no: int | None = None
+        column: str | None = None
+        if len(path) >= 1 and isinstance(path[0], str):
+            table_id = path[0]
+        if len(path) >= 2 and isinstance(path[1], int):
+            row_no = path[1]
+        if len(path) >= 3 and isinstance(path[2], str):
+            column = path[2]
+            column = self.field_to_column(table_id, column)
+        if isinstance(row_no, int):
+            row_no = self.adjust_row_number(table_id, row_no)
+        location_parts = []
+        if table_id is not None:
+            location_parts.append(f"table {table_id!r}")
+        if row_no is not None:
+            location_parts.append(f"row {row_no}")
+        if column is not None:
+            location_parts.append(f"column {column!r}")
+        if len(path) >= 4:
+            location_parts.append("-> " + ".".join(str(p) for p in path[3:]))
+
+        return " ".join(location_parts)
+
+    def adjust_row_number(self, table_id: str | None, row_no: int) -> int:
+        table_read = table_id and self.table_read.get(table_id)
+        if table_read:
+            return table_read.adjusted_row_number(row_no)
+        return row_no + 1  # Convert to 1-indexed if no table read info is available
+
+    @classmethod
+    def field_to_column(cls, table_id: str | None, field_: str) -> str:
+        """Maps the field name used in the DMS API to the column named used by Neat."""
+        mapping = cls.field_mapping(table_id)
+        if mapping is not None:
+            return mapping.get(field_, field_)
+        return field_
+
+    @classmethod
+    def field_mapping(cls, table_id: str | int | None) -> Mapping[str, str] | None:
+        if not isinstance(table_id, str):
+            return None
+        return DMS_API_MAPPING.get(table_id)

cognite/neat/_data_model/models/conceptual/_base.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, Field
+from pydantic.alias_generators import to_camel
+
+from cognite.neat._data_model._identifiers import URI
+
+
+class BaseModelObject(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
+    """Base class for all object. This includes resources and nested objects."""
+
+    ...
+
+
+class ResourceMetadata(BaseModelObject):
+    name: str | None = Field(
+        None, description="Human readable / display name of resource being described.", max_length=1024
+    )
+    description: str | None = Field(None, description="The description of the resource.", max_length=255)
+    uri: URI | None = Field(None, description="The URI of the resource being described.")

cognite/neat/_data_model/models/conceptual/_concept.py

@@ -0,0 +1,67 @@
+from collections import Counter
+
+from pydantic import Field, ValidationInfo, field_validator
+from pyparsing import cast
+
+from cognite.neat._data_model.models.entities import ConceptEntity
+from cognite.neat._data_model.models.entities._constants import PREFIX_PATTERN, SUFFIX_PATTERN, VERSION_PATTERN
+from cognite.neat._utils.text import humanize_collection
+
+from ._base import ResourceMetadata
+from ._property import Property
+
+
+class Concept(ResourceMetadata):
+    space: str = Field(
+        description="Id of the space that the concept belongs to.",
+        min_length=1,
+        max_length=43,
+        pattern=PREFIX_PATTERN,
+        alias="prefix",
+    )
+    external_id: str = Field(
+        description="External-id of the concept.",
+        min_length=1,
+        max_length=255,
+        pattern=SUFFIX_PATTERN,
+        alias="suffix",
+    )
+    version: str | None = Field(
+        default=None,
+        description="Version of the concept.",
+        max_length=43,
+        pattern=VERSION_PATTERN,
+    )
+
+    implements: list[ConceptEntity] | None = Field(
+        default=None,
+        description="References to the concepts from where this concept will inherit properties.",
+    )
+
+    properties: dict[str, Property] | None = Field(default=None, description="Properties associated with the concept.")
+
+    @field_validator("implements", mode="after")
+    def cannot_implement_itself(cls, value: list[ConceptEntity], info: ValidationInfo) -> list[ConceptEntity]:
+        if not value:
+            return value
+
+        this_concept = ConceptEntity(
+            prefix=info.data["space"],
+            suffix=info.data["external_id"],
+            version=cast(str, info.data.get("version")),
+        )
+
+        if this_concept in value:
+            raise ValueError("A concept cannot implement itself.")
+
+        return value
+
+    @field_validator("implements", mode="after")
+    def cannot_have_duplicates(cls, value: list[ConceptEntity], info: ValidationInfo) -> list[ConceptEntity]:
+        counts = Counter(value)
+        duplicates = {concept for concept, count in counts.items() if count > 1}
+
+        if duplicates:
+            raise ValueError(f"Duplicate concepts found: {humanize_collection(duplicates)}")
+
+        return value

cognite/neat/_data_model/models/conceptual/_data_model.py

@@ -0,0 +1,51 @@
+from collections import Counter
+
+from pydantic import Field, ValidationInfo, field_validator
+
+from cognite.neat._data_model.models.entities import ConceptEntity
+from cognite.neat._data_model.models.entities._constants import PREFIX_PATTERN, SUFFIX_PATTERN, VERSION_PATTERN
+from cognite.neat._utils.text import humanize_collection
+
+from ._base import ResourceMetadata
+from ._concept import Concept
+
+
+class DataModel(ResourceMetadata):
+    space: str = Field(
+        description="Id of the space that the concept belongs to.",
+        min_length=1,
+        max_length=43,
+        pattern=PREFIX_PATTERN,
+        alias="prefix",
+    )
+    external_id: str = Field(
+        description="External-id of the concept.",
+        min_length=1,
+        max_length=255,
+        pattern=SUFFIX_PATTERN,
+        alias="suffix",
+    )
+    version: str = Field(
+        description="Version of the concept.",
+        max_length=43,
+        pattern=VERSION_PATTERN,
+    )
+
+    concepts: list[Concept] = Field(
+        description="References to the concepts from where this concept will inherit properties.",
+    )
+
+    @field_validator("concepts", mode="after")
+    def cannot_have_duplicates(cls, value: list[Concept], info: ValidationInfo) -> list[Concept]:
+        concept_ids = [
+            ConceptEntity(prefix=concept.space, suffix=concept.external_id, version=concept.version)
+            for concept in value
+        ]
+
+        counts = Counter(concept_ids)
+        duplicates = {concept for concept, count in counts.items() if count > 1}
+
+        if duplicates:
+            raise ValueError(f"Duplicate concepts found: {humanize_collection(duplicates)}")
+
+        return value

cognite/neat/_data_model/models/conceptual/_properties.py

@@ -0,0 +1,104 @@
+from typing import Any
+
+from pydantic import Field, ValidationInfo, field_validator, model_validator
+
+from cognite.neat._data_model.models.entities import URI, ConceptEntity, DataType, UnknownEntity
+
+from ._base import ResourceMetadata
+
+
+class Property(ResourceMetadata):
+    value_type: DataType | ConceptEntity | UnknownEntity = Field(
+        union_mode="left_to_right",
+        description="Value type that the property can hold. It takes either subset of XSD type or a class defined.",
+    )
+    min_count: int | None = Field(
+        default=None,
+        ge=0,
+        description="Minimum number of values that the property can hold. "
+        "If no value is provided, the default value is None meaning `0`, "
+        "which means that the property is optional.",
+    )
+    max_count: int | None = Field(
+        default=None,
+        ge=0,
+        description="Maximum number of values that the property can hold. "
+        "If no value is provided, the default value is None meaning `inf`, "
+        "which means that the property can hold any number of values (listable).",
+    )
+    default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
+    instance_reference: list[URI] | None = Field(
+        default=None,
+        description="The URI(s) in the graph to get the value of the property.",
+    )
+
+    @model_validator(mode="after")
+    def check_min_max_count(self) -> "Property":
+        if self.min_count is not None and self.max_count is not None:
+            if self.min_count > self.max_count:
+                raise ValueError("min_count must be less than or equal to max_count")
+        return self
+
+    @field_validator("default", mode="after")
+    def check_default_value_primitive_type(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+        if not isinstance(value_type, DataType):
+            raise ValueError("Setting default value is only supported for primitive value types.")
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_python_type_exists(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+
+        if isinstance(value_type, DataType) and not hasattr(value_type, "python"):
+            raise ValueError(
+                f"DataType {value_type} does not have a python type defined."
+                " Setting default value for property is not possible."
+            )
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_not_list(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        if isinstance(value, list):
+            raise ValueError("Setting list as default value is not supported.")
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_single_valued(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        max_count = info.data.get("max_count")
+
+        if max_count is None or max_count > 1:
+            raise ValueError(
+                "Setting default value is only supported for single-valued properties."
+                f" Property has max_count={max_count or 'Inf'}."
+            )
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_type_match(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+
+        if (
+            isinstance(value_type, DataType)
+            and hasattr(value_type, "python")
+            and not isinstance(value, value_type.python)
+        ):
+            raise ValueError(
+                f"Default value type is {type(value)}, which does not match expected value type {value_type.python}."
+            )
+        return value

cognite/neat/_data_model/models/conceptual/_property.py

@@ -0,0 +1,105 @@
+from typing import Any
+
+from pydantic import Field, ValidationInfo, field_validator, model_validator
+
+from cognite.neat._data_model.models.entities import URI, ConceptEntity, DataType, UnknownEntity
+
+from ._base import ResourceMetadata
+
+
+class Property(ResourceMetadata):
+    value_type: DataType | ConceptEntity | UnknownEntity = Field(
+        union_mode="left_to_right",
+        description="Value type that the property can hold. It takes either subset of XSD type or a class defined.",
+    )
+    min_count: int | None = Field(
+        default=None,
+        ge=0,
+        description="Minimum number of values that the property can hold. "
+        "If no value is provided, the default value is None meaning `0`, "
+        "which means that the property is optional.",
+    )
+    max_count: int | None = Field(
+        default=None,
+        ge=0,
+        description="Maximum number of values that the property can hold. "
+        "If no value is provided, the default value is None meaning `inf`, "
+        "which means that the property can hold any number of values (listable).",
+    )
+    default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
+    instance_reference: list[URI] | None = Field(
+        default=None,
+        description="The URI(s) in the graph to get the value of the property.",
+    )
+
+    @model_validator(mode="after")
+    def check_min_max_count(self) -> "Property":
+        if self.min_count is not None and self.max_count is not None:
+            if self.min_count > self.max_count:
+                raise ValueError("min_count must be less than or equal to max_count")
+        return self
+
+    @field_validator("default", mode="after")
+    def check_default_value_primitive_type(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+        if not isinstance(value_type, DataType):
+            raise ValueError("Setting default value is only supported for primitive value types.")
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_python_type_exists(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+
+        if isinstance(value_type, DataType) and not hasattr(value_type, "python"):
+            raise ValueError(
+                f"DataType {value_type} does not have a python type defined."
+                " Setting default value for property is not possible."
+            )
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_not_list(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        if isinstance(value, list):
+            raise ValueError("Setting list as default value is not supported.")
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_single_valued(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        max_count = info.data.get("max_count")
+        min_count = info.data.get("min_count")
+
+        if max_count is None or max_count > 1 or (min_count and min_count > 1):
+            raise ValueError(
+                "Setting default value is only supported for single-valued properties."
+                f" Property has min_count={info.data.get('min_count')} and max_count={info.data.get('max_count')}."
+            )
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_type_match(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+
+        if (
+            isinstance(value_type, DataType)
+            and hasattr(value_type, "python")
+            and not isinstance(value, value_type.python)
+        ):
+            raise ValueError(
+                f"Default value type is {type(value)}, which does not match expected value type {value_type.python}."
+            )
+        return value

cognite/neat/_data_model/models/dms/__init__.py

@@ -0,0 +1,206 @@
+from cognite.neat._data_model.models.dms._base import (
+    APIResource,
+    BaseModelObject,
+    Resource,
+    T_Resource,
+    WriteableResource,
+)
+from cognite.neat._data_model.models.dms._constraints import (
+    Constraint,
+    ConstraintAdapter,
+    ConstraintDefinition,
+    RequiresConstraintDefinition,
+    UniquenessConstraintDefinition,
+)
+from cognite.neat._data_model.models.dms._container import (
+    Container,
+    ContainerPropertyDefinition,
+    ContainerRequest,
+    ContainerResponse,
+)
+from cognite.neat._data_model.models.dms._data_types import (
+    DMS_DATA_TYPES,
+    BooleanProperty,
+    DataType,
+    DataTypeAdapter,
+    DateProperty,
+    DirectNodeRelation,
+    EnumProperty,
+    EnumValue,
+    FileCDFExternalIdReference,
+    Float32Property,
+    Float64Property,
+    FloatProperty,
+    Int32Property,
+    Int64Property,
+    JSONProperty,
+    ListablePropertyTypeDefinition,
+    PropertyTypeDefinition,
+    SequenceCDFExternalIdReference,
+    TextProperty,
+    TimeseriesCDFExternalIdReference,
+    TimestampProperty,
+)
+from cognite.neat._data_model.models.dms._indexes import BtreeIndex, Index, IndexAdapter, IndexDefinition, InvertedIndex
+from cognite.neat._data_model.models.dms._space import Space, SpaceRequest, SpaceResponse
+
+from ._data_model import DataModelRequest, DataModelResponse
+from ._http import DataModelBody, DataModelResource, ResourceId, T_DataModelResource, T_ResourceId
+from ._references import (
+    ContainerConstraintReference,
+    ContainerDirectReference,
+    ContainerIndexReference,
+    ContainerReference,
+    DataModelReference,
+    NodeReference,
+    SpaceReference,
+    ViewDirectReference,
+    ViewReference,
+)
+from ._schema import RequestSchema
+from ._view_filter import (
+    AVAILABLE_FILTERS,
+    AndFilter,
+    ContainsAllFilterData,
+    ContainsAnyFilterData,
+    EqualsFilterData,
+    ExistsFilterData,
+    Filter,
+    FilterAdapter,
+    FilterDataDefinition,
+    HasDataFilter,
+    InFilterData,
+    InstanceReferencesFilterData,
+    MatchAllFilterData,
+    NestedFilterData,
+    NotFilter,
+    OrFilter,
+    OverlapsFilterData,
+    PrefixFilterData,
+    RangeFilterData,
+)
+from ._view_property import (
+    ConnectionPropertyDefinition,
+    ConstraintOrIndexState,
+    MultiEdgeProperty,
+    MultiReverseDirectRelationPropertyRequest,
+    MultiReverseDirectRelationPropertyResponse,
+    SingleEdgeProperty,
+    SingleReverseDirectRelationPropertyRequest,
+    SingleReverseDirectRelationPropertyResponse,
+    ViewCorePropertyRequest,
+    ViewCorePropertyResponse,
+    ViewPropertyDefinition,
+    ViewRequestProperty,
+    ViewRequestPropertyAdapter,
+    ViewResponseProperty,
+)
+from ._views import (
+    View,
+    ViewRequest,
+    ViewResponse,
+)
+
+__all__ = [
+    "AVAILABLE_FILTERS",
+    "DMS_DATA_TYPES",
+    "APIResource",
+    "AndFilter",
+    "BaseModelObject",
+    "BooleanProperty",
+    "BtreeIndex",
+    "ConnectionPropertyDefinition",
+    "Constraint",
+    "ConstraintAdapter",
+    "ConstraintDefinition",
+    "ConstraintOrIndexState",
+    "Container",
+    "ContainerConstraintReference",
+    "ContainerDirectReference",
+    "ContainerIndexReference",
+    "ContainerPropertyDefinition",
+    "ContainerReference",
+    "ContainerRequest",
+    "ContainerResponse",
+    "ContainsAllFilterData",
+    "ContainsAnyFilterData",
+    "DataModelBody",
+    "DataModelReference",
+    "DataModelRequest",
+    "DataModelResource",
+    "DataModelResponse",
+    "DataType",
+    "DataTypeAdapter",
+    "DateProperty",
+    "DirectNodeRelation",
+    "EnumProperty",
+    "EnumValue",
+    "EqualsFilterData",
+    "ExistsFilterData",
+    "FileCDFExternalIdReference",
+    "Filter",
+    "FilterAdapter",
+    "FilterDataDefinition",
+    "Float32Property",
+    "Float64Property",
+    "FloatProperty",
+    "HasDataFilter",
+    "InFilterData",
+    "Index",
+    "IndexAdapter",
+    "IndexDefinition",
+    "InstanceReferencesFilterData",
+    "Int32Property",
+    "Int64Property",
+    "InvertedIndex",
+    "JSONProperty",
+    "ListablePropertyTypeDefinition",
+    "MatchAllFilterData",
+    "MultiEdgeProperty",
+    "MultiReverseDirectRelationPropertyRequest",
+    "MultiReverseDirectRelationPropertyResponse",
+    "NestedFilterData",
+    "NodeReference",
+    "NotFilter",
+    "OrFilter",
+    "OverlapsFilterData",
+    "PrefixFilterData",
+    "PropertyTypeDefinition",
+    "RangeFilterData",
+    "RequestSchema",
+    "RequiresConstraintDefinition",
+    "Resource",
+    "ResourceId",
+    "SequenceCDFExternalIdReference",
+    "SequenceCDFExternalIdReference",
+    "SingleEdgeProperty",
+    "SingleReverseDirectRelationPropertyRequest",
+    "SingleReverseDirectRelationPropertyResponse",
+    "Space",
+    "SpaceReference",
+    "SpaceRequest",
+    "SpaceResponse",
+    "T_DataModelResource",
+    "T_Resource",
+    "T_ResourceId",
+    "TextProperty",
+    "TextProperty",
+    "TimeseriesCDFExternalIdReference",
+    "TimeseriesCDFExternalIdReference",
+    "TimestampProperty",
+    "TimestampProperty",
+    "UniquenessConstraintDefinition",
+    "UniquenessConstraintDefinition",
+    "View",
+    "ViewCorePropertyRequest",
+    "ViewCorePropertyResponse",
+    "ViewDirectReference",
+    "ViewPropertyDefinition",
+    "ViewReference",
+    "ViewRequest",
+    "ViewRequestProperty",
+    "ViewRequestPropertyAdapter",
+    "ViewResponse",
+    "ViewResponseProperty",
+    "WriteableResource",
+]

cognite/neat/_data_model/models/dms/_base.py

@@ -0,0 +1,31 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from cognite.neat._utils.useful_types import BaseModelObject, T_Reference
+
+
+class Resource(BaseModelObject):
+    """Base class for all data modeling resources."""
+
+    ...
+
+
+T_Resource = TypeVar("T_Resource", bound=Resource)
+
+
+class WriteableResource(Resource, Generic[T_Resource], ABC):
+    """Base class for all writeable data modeling resources."""
+
+    @abstractmethod
+    def as_request(self) -> T_Resource:
+        """Convert the response model to a request model by removing read-only fields."""
+        raise NotImplementedError()
+
+
+class APIResource(Generic[T_Reference], ABC):
+    """Base class for all API data modeling resources."""
+
+    @abstractmethod
+    def as_reference(self) -> T_Reference:
+        """Convert the resource to a reference object (identifier)."""
+        raise NotImplementedError()