cognite-neat 0.123.35__tar.gz → 0.123.37__tar.gz

cognite_neat-0.123.37/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+* @cognitedata/accelerate
+

{cognite_neat-0.123.35 → cognite_neat-0.123.37}/.github/workflows/code-quality.yaml

@@ -45,7 +45,6 @@ jobs:
 
   checks:
     runs-on: ubuntu-latest
-    environment: dev
     strategy:
       fail-fast: false
       matrix:
@@ -70,7 +69,7 @@ jobs:
       - run: uv sync --all-extras
 
       - name: Run unit tests
-        run: export PYTHONPATH=$PYTHONPATH:$(pwd); uv run pytest tests/tests_unit
+        run: export PYTHONPATH=$PYTHONPATH:$(pwd); uv run pytest tests/tests_unit tests/tests_unit_v1
 
   build-docs:
     runs-on: ubuntu-latest
@@ -92,9 +91,9 @@ jobs:
           mkdocs build
 
 
+
   coverage:
     runs-on: ubuntu-latest
-    environment: dev
     steps:
       - uses: actions/checkout@v3
 
@@ -117,7 +116,7 @@ jobs:
           CDF_PROJECT: ${{ vars.CDF_PROJECT }}
           IDP_CLIENT_ID: ${{ vars.IDP_CLIENT_ID }}
           IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }}
-          IDP_TENANT_ID: ${{ secrets.IDP_TENANT_ID }}
+          IDP_TENANT_ID: ${{ vars.IDP_TENANT_ID }}
         run: export PYTHONPATH=$PYTHONPATH:$(pwd); uv run pytest --cov=cognite/ --cov-config=pyproject.toml --cov-report=xml:coverage.xml tests/
 
       - name: Push coverage report to PR
@@ -126,6 +125,11 @@ jobs:
             coverageFile: coverage.xml
             token: ${{ secrets.GITHUB_TOKEN }}
             thresholdAll: 0.8
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: cognitedata/neat
 
   validate-description:
     name: Validate PR description

{cognite_neat-0.123.35 → cognite_neat-0.123.37}/.github/workflows/release.yaml

@@ -10,6 +10,40 @@ env:
   PYTHON_VERSION: "3.11"
 
 jobs:
+  upload_coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Set up cache
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('uv.lock') }}
+
+      - run: uv sync --all-extras
+
+      - name: Create test coverage report
+        env:
+          CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
+          CDF_PROJECT: ${{ vars.CDF_PROJECT }}
+          IDP_CLIENT_ID: ${{ vars.IDP_CLIENT_ID }}
+          IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }}
+          IDP_TENANT_ID: ${{ vars.IDP_TENANT_ID }}
+        run: export PYTHONPATH=$PYTHONPATH:$(pwd); uv run pytest --cov=cognite/ --cov-config=pyproject.toml --cov-report=xml:coverage.xml tests/
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: cognitedata/neat
+
+
   release-to-pypi:
     runs-on: ubuntu-latest
     environment: CD

{cognite_neat-0.123.35 → cognite_neat-0.123.37}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.123.35
+Version: 0.123.37
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: GitHub, https://github.com/cognitedata/neat
 Project-URL: Changelog, https://github.com/cognitedata/neat/releases
-Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>, Rogerio Júnior <rogerio.junior@cognite.com>
+Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>
 License-Expression: Apache-2.0
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -96,16 +96,15 @@ The plot below shows the NEAT authorship from the start until present day.
 #### Current authors
 - [Nikola Vasiljević](www.linkedin.com/in/thisisnikola)
 - [Anders Albert](https://www.linkedin.com/in/anders-albert-00790483/)
-- [Rogerio Júnior](https://www.linkedin.com/in/rogerio-saboia-j%C3%BAnior-087118a7/)
 
 #### Former authors
 - [Aleksandrs Livincovs](https://www.linkedin.com/in/aleksandrslivincovs/)
 - [Julia Graham](https://www.linkedin.com/in/julia-graham-959a78a7/)
+- [Rogerio Júnior](https://www.linkedin.com/in/rogerio-saboia-j%C3%BAnior-087118a7/)
 
 ### Contributors
 We are very grateful for the contributions made by:
 
-
 - [Marie Solvik Lepoutre](https://www.linkedin.com/in/mslepoutre/), who improved RDF triples projections to Cognite Data Fusion
 - [Bård Henning Tvedt](https://www.linkedin.com/in/bhtvedt/), who implemented IMF importer
 - [Hassan Gomaa](https://www.linkedin.com/in/dr-hassan-gomaa-232638121/), who extended the DEXPI extractor

{cognite_neat-0.123.35 → cognite_neat-0.123.37}/README.md

@@ -41,16 +41,15 @@ The plot below shows the NEAT authorship from the start until present day.
 #### Current authors
 - [Nikola Vasiljević](www.linkedin.com/in/thisisnikola)
 - [Anders Albert](https://www.linkedin.com/in/anders-albert-00790483/)
-- [Rogerio Júnior](https://www.linkedin.com/in/rogerio-saboia-j%C3%BAnior-087118a7/)
 
 #### Former authors
 - [Aleksandrs Livincovs](https://www.linkedin.com/in/aleksandrslivincovs/)
 - [Julia Graham](https://www.linkedin.com/in/julia-graham-959a78a7/)
+- [Rogerio Júnior](https://www.linkedin.com/in/rogerio-saboia-j%C3%BAnior-087118a7/)
 
 ### Contributors
 We are very grateful for the contributions made by:
 
-
 - [Marie Solvik Lepoutre](https://www.linkedin.com/in/mslepoutre/), who improved RDF triples projections to Cognite Data Fusion
 - [Bård Henning Tvedt](https://www.linkedin.com/in/bhtvedt/), who implemented IMF importer
 - [Hassan Gomaa](https://www.linkedin.com/in/dr-hassan-gomaa-232638121/), who extended the DEXPI extractor

cognite_neat-0.123.37/cognite/neat/_data_model/models/conceptual/_base.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, Field
+from pydantic.alias_generators import to_camel
+
+from cognite.neat._data_model._identifiers import URI
+
+
+class BaseModelObject(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
+    """Base class for all object. This includes resources and nested objects."""
+
+    ...
+
+
+class ResourceMetadata(BaseModelObject):
+    name: str | None = Field(
+        None, description="Human readable / display name of resource being described.", max_length=1024
+    )
+    description: str | None = Field(None, description="The description of the resource.", max_length=255)
+    uri: URI | None = Field(None, description="The URI of the resource being described.")

cognite_neat-0.123.37/cognite/neat/_data_model/models/conceptual/_properties.py

@@ -0,0 +1,104 @@
+from typing import Any
+
+from pydantic import Field, ValidationInfo, field_validator, model_validator
+
+from cognite.neat._data_model.models.entities import URI, ConceptEntity, DataType, UnknownEntity
+
+from ._base import ResourceMetadata
+
+
+class Property(ResourceMetadata):
+    value_type: DataType | ConceptEntity | UnknownEntity = Field(
+        union_mode="left_to_right",
+        description="Value type that the property can hold. It takes either subset of XSD type or a class defined.",
+    )
+    min_count: int | None = Field(
+        default=None,
+        ge=0,
+        description="Minimum number of values that the property can hold. "
+        "If no value is provided, the default value is None meaning `0`, "
+        "which means that the property is optional.",
+    )
+    max_count: int | None = Field(
+        default=None,
+        ge=0,
+        description="Maximum number of values that the property can hold. "
+        "If no value is provided, the default value is None meaning `inf`, "
+        "which means that the property can hold any number of values (listable).",
+    )
+    default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
+    instance_reference: list[URI] | None = Field(
+        default=None,
+        description="The URI(s) in the graph to get the value of the property.",
+    )
+
+    @model_validator(mode="after")
+    def check_min_max_count(self) -> "Property":
+        if self.min_count is not None and self.max_count is not None:
+            if self.min_count > self.max_count:
+                raise ValueError("min_count must be less than or equal to max_count")
+        return self
+
+    @field_validator("default", mode="after")
+    def check_default_value_primitive_type(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+        if not isinstance(value_type, DataType):
+            raise ValueError("Setting default value is only supported for primitive value types.")
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_python_type_exists(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+
+        if isinstance(value_type, DataType) and not hasattr(value_type, "python"):
+            raise ValueError(
+                f"DataType {value_type} does not have a python type defined."
+                " Setting default value for property is not possible."
+            )
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_not_list(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        if isinstance(value, list):
+            raise ValueError("Setting list as default value is not supported.")
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_single_valued(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        max_count = info.data.get("max_count")
+
+        if max_count is None or max_count > 1:
+            raise ValueError(
+                "Setting default value is only supported for single-valued properties."
+                f" Property has max_count={max_count or 'Inf'}."
+            )
+        return value
+
+    @field_validator("default", mode="after")
+    def check_default_value_type_match(cls, value: Any, info: ValidationInfo) -> Any:
+        if not value:
+            return value
+
+        value_type = info.data.get("value_type")
+
+        if (
+            isinstance(value_type, DataType)
+            and hasattr(value_type, "python")
+            and not isinstance(value, value_type.python)
+        ):
+            raise ValueError(
+                f"Default value type is {type(value)}, which does not match expected value type {value_type.python}."
+            )
+        return value

cognite_neat-0.123.37/cognite/neat/_data_model/models/dms/__init__.py

@@ -0,0 +1,69 @@
+from cognite.neat._data_model.models.dms._base import Resource, WriteableResource
+from cognite.neat._data_model.models.dms._constraints import (
+    Constraint,
+    ConstraintDefinition,
+    RequiresConstraintDefinition,
+    UniquenessConstraintDefinition,
+)
+from cognite.neat._data_model.models.dms._container import (
+    Container,
+    ContainerPropertyDefinition,
+    ContainerRequest,
+    ContainerResponse,
+)
+from cognite.neat._data_model.models.dms._data_types import (
+    BooleanProperty,
+    DataType,
+    DateProperty,
+    DirectNodeRelation,
+    EnumProperty,
+    FileCDFExternalIdReference,
+    FloatProperty,
+    Int32Property,
+    Int64Property,
+    JSONProperty,
+    ListablePropertyTypeDefinition,
+    PropertyTypeDefinition,
+    SequenceCDFExternalIdReference,
+    TextProperty,
+    TimeseriesCDFExternalIdReference,
+    TimestampProperty,
+)
+from cognite.neat._data_model.models.dms._indexes import BtreeIndex, Index, IndexDefinition, InvertedIndex
+from cognite.neat._data_model.models.dms._space import Space, SpaceRequest, SpaceResponse
+
+__all__ = [
+    "BooleanProperty",
+    "BtreeIndex",
+    "Constraint",
+    "ConstraintDefinition",
+    "Container",
+    "ContainerPropertyDefinition",
+    "ContainerRequest",
+    "ContainerResponse",
+    "DataType",
+    "DateProperty",
+    "DirectNodeRelation",
+    "EnumProperty",
+    "FileCDFExternalIdReference",
+    "FloatProperty",
+    "Index",
+    "IndexDefinition",
+    "Int32Property",
+    "Int64Property",
+    "InvertedIndex",
+    "JSONProperty",
+    "ListablePropertyTypeDefinition",
+    "PropertyTypeDefinition",
+    "RequiresConstraintDefinition",
+    "Resource",
+    "SequenceCDFExternalIdReference",
+    "Space",
+    "SpaceRequest",
+    "SpaceResponse",
+    "TextProperty",
+    "TimeseriesCDFExternalIdReference",
+    "TimestampProperty",
+    "UniquenessConstraintDefinition",
+    "WriteableResource",
+]

cognite_neat-0.123.37/cognite/neat/_data_model/models/dms/_constants.py

@@ -0,0 +1,46 @@
+SPACE_FORMAT_PATTERN = r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$"
+DM_EXTERNAL_ID_PATTERN = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
+CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN = r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,253}[a-zA-Z0-9]?$"
+INSTANCE_ID_PATTERN = r"^[^\\x00]{1,256}$"
+ENUM_VALUE_IDENTIFIER_PATTERN = r"^[_A-Za-z][_0-9A-Za-z]{0,127}$"
+FORBIDDEN_ENUM_VALUES = frozenset({"true", "false", "null"})
+FORBIDDEN_SPACES = frozenset(["space", "cdf", "dms", "pg3", "shared", "system", "node", "edge"])
+FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS = frozenset(
+    [
+        "Query",
+        "Mutation",
+        "Subscription",
+        "String",
+        "Int32",
+        "Int64",
+        "Int",
+        "Float32",
+        "Float64",
+        "Float",
+        "Timestamp",
+        "JSONObject",
+        "Date",
+        "Numeric",
+        "Boolean",
+        "PageInfo",
+        "File",
+        "Sequence",
+        "TimeSeries",
+    ]
+)
+FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER = frozenset(
+    [
+        "space",
+        "externalId",
+        "createdTime",
+        "lastUpdatedTime",
+        "deletedTime",
+        "edge_id",
+        "node_id",
+        "project_id",
+        "property_group",
+        "seq",
+        "tg_table_name",
+        "extensions",
+    ]
+)

cognite_neat-0.123.37/cognite/neat/_data_model/models/dms/_constraints.py

@@ -0,0 +1,30 @@
+from abc import ABC
+from typing import Annotated, Literal
+
+from pydantic import Field
+
+from ._base import BaseModelObject
+from ._references import ContainerReference
+
+
+class ConstraintDefinition(BaseModelObject, ABC):
+    constraint_type: str
+
+
+class UniquenessConstraintDefinition(ConstraintDefinition):
+    constraint_type: Literal["uniqueness"] = "uniqueness"
+    properties: list[str] = Field(
+        description="List of properties included in the constraint.", min_length=1, max_length=10
+    )
+    by_space: bool | None = Field(default=None, description="Whether to make the constraint space-specific.")
+
+
+class RequiresConstraintDefinition(ConstraintDefinition):
+    constraint_type: Literal["requires"] = "requires"
+    require: ContainerReference = Field(description="Reference to an existing container.")
+
+
+Constraint = Annotated[
+    UniquenessConstraintDefinition | RequiresConstraintDefinition,
+    Field(discriminator="constraint_type"),
+]

cognite_neat-0.123.37/cognite/neat/_data_model/models/dms/_container.py

@@ -0,0 +1,152 @@
+import re
+from abc import ABC
+from typing import Any, Literal
+
+from pydantic import Field, field_validator
+from pydantic_core.core_schema import ValidationInfo
+
+from cognite.neat._utils.text import humanize_collection
+
+from ._base import BaseModelObject, Resource, WriteableResource
+from ._constants import (
+    CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN,
+    DM_EXTERNAL_ID_PATTERN,
+    FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS,
+    FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER,
+    SPACE_FORMAT_PATTERN,
+)
+from ._constraints import Constraint
+from ._data_types import DataType
+from ._indexes import Index
+
+KEY_PATTERN = re.compile(CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN)
+
+
+class ContainerPropertyDefinition(BaseModelObject):
+    immutable: bool | None = Field(
+        default=None,
+        description="Should updates to this property be rejected after the initial population?",
+    )
+    nullable: bool | None = Field(
+        default=None,
+        description="Does this property need to be set to a value, or not?",
+    )
+    auto_increment: bool | None = Field(
+        default=None,
+        description="Increment the property based on its highest current value (max value).",
+    )
+    default_value: str | int | bool | dict[str, Any] | None = Field(
+        default=None,
+        description="Default value to use when you do not specify a value for the property.",
+    )
+    description: str | None = Field(
+        default=None,
+        description="Description of the content and suggested use for this property.",
+        max_length=1024,
+    )
+    name: str | None = Field(
+        default=None,
+        description="Readable property name.",
+        max_length=255,
+    )
+    type: DataType = Field(description="The type of data you can store in this property.")
+
+
+class Container(Resource, ABC):
+    space: str = Field(
+        description="The workspace for the container, a unique identifier for the space.",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    external_id: str = Field(
+        description="External-id of the container.",
+        min_length=1,
+        max_length=255,
+        pattern=DM_EXTERNAL_ID_PATTERN,
+    )
+    name: str | None = Field(
+        default=None,
+        description="name for the container.",
+        max_length=255,
+    )
+    description: str | None = Field(
+        default=None,
+        description="Description of the container.",
+        max_length=1024,
+    )
+    used_for: Literal["node", "edge", "all"] | None = Field(
+        default=None,
+        description="Should this operation apply to nodes, edges or both.",
+    )
+    properties: dict[str, ContainerPropertyDefinition] = Field(
+        description="Set of properties to apply to the container.",
+        min_length=1,
+    )
+    constraints: dict[str, Constraint] | None = Field(
+        default=None,
+        description="Set of constraints to apply to the container.",
+        max_length=10,
+    )
+    indexes: dict[str, Index] | None = Field(
+        default=None,
+        description="Set of indexes to apply to the container.",
+        max_length=10,
+    )
+
+    @field_validator("indexes", "constraints", mode="after")
+    def validate_key_length(cls, val: dict[str, Any] | None, info: ValidationInfo) -> dict[str, Any] | None:
+        """Validate keys"""
+        if not isinstance(val, dict):
+            return val
+        invalid_keys = {key for key in val.keys() if not (1 <= len(key) <= 43)}
+        if invalid_keys:
+            raise ValueError(
+                f"{info.field_name} keys must be between 1 and 43 characters long. Invalid keys: "
+                f"{humanize_collection(invalid_keys)}"
+            )
+        return val
+
+    @field_validator("properties", mode="after")
+    def validate_property_keys(cls, val: dict[str, Any]) -> dict[str, Any]:
+        """Validate property keys"""
+        if invalid_keys := {key for key in val if not KEY_PATTERN.match(key)}:
+            raise ValueError(
+                f"Property keys must match pattern '{CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER_PATTERN}'. "
+                f"Invalid keys: {humanize_collection(invalid_keys)}"
+            )
+
+        if forbidden_keys := set(val.keys()).intersection(FORBIDDEN_CONTAINER_AND_VIEW_PROPERTIES_IDENTIFIER):
+            raise ValueError(
+                f"Property keys cannot be any of the following reserved values: {humanize_collection(forbidden_keys)}"
+            )
+        return val
+
+    @field_validator("external_id")
+    def check_forbidden_external_id_value(cls, val: str) -> str:
+        """Check the external_id not present in forbidden set"""
+        if val in FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS:
+            raise ValueError(
+                f"{val!r} is a reserved container External ID. Reserved External IDs are: "
+                f"{humanize_collection(FORBIDDEN_CONTAINER_AND_VIEW_EXTERNAL_IDS)}"
+            )
+        return val
+
+
+class ContainerRequest(Container): ...
+
+
+class ContainerResponse(Container, WriteableResource[ContainerRequest]):
+    created_time: int = Field(
+        description="When the container was created. The number of milliseconds since 00:00:00 "
+        "Thursday, 1 January 1970, "
+        "Coordinated Universal Time (UTC), minus leap seconds."
+    )
+    last_updated_time: int = Field(
+        description="When the container was last updated. The number of milliseconds since 00:00:00 Thursday, "
+        "1 January 1970, Coordinated Universal Time (UTC), minus leap seconds."
+    )
+    is_global: bool = Field(description="Whether the container is a global container.")
+
+    def as_request(self) -> "ContainerRequest":
+        return ContainerRequest.model_validate(self.model_dump(by_alias=True))