cognite-neat 0.123.43__py3-none-any.whl → 0.125.0__py3-none-any.whl

cognite/neat/_exceptions.py

@@ -0,0 +1,17 @@
+from ._issues import ModelSyntaxError
+
+
+class NeatException(Exception):
+    """Base class for all exceptions raised by Neat."""
+
+    pass
+
+
+class DataModelImportError(NeatException):
+    """Raised when there is an error importing a model."""
+
+    def __init__(self, errors: list[ModelSyntaxError]) -> None:
+        self.errors = errors
+
+    def __str__(self) -> str:
+        return f"Model import failed with {len(self.errors)} errors: " + "; ".join(map(str, self.errors))

cognite/neat/_session/_session.py

@@ -0,0 +1,33 @@
+from ._state_machine import EmptyState, ForbiddenState, State
+
+
+class NeatSession:
+    """A session is an interface for neat operations. It works as
+    a manager for handling user interactions and orchestrating
+    the state machine for data model and instance operations.
+    """
+
+    def __init__(self) -> None:
+        self.state: State = EmptyState()
+
+    def _execute_event(self, event: str) -> bool:
+        """Place holder function for executing events and transitioning states.
+        It will be modified to include actual logic as we progress with v1 of neat.
+
+        """
+        print(f"\n--- Executing event: '{event}' from {self.state} ---")
+
+        old_state = self.state
+        new_state = self.state.on_event(event)
+
+        # Handle ForbiddenState
+        if isinstance(new_state, ForbiddenState):
+            print(f"❌ Event '{event}' is FORBIDDEN from {old_state}")
+            # Return to previous state (as per your table logic)
+            self.state = new_state.on_event("undo")
+            print(f"↩️  Returned to: {self.state}")
+            return False
+        else:
+            self.state = new_state
+            print(f"✅ Transition successful: {old_state} → {self.state}")
+            return True

cognite/neat/_session/_state_machine/__init__.py

@@ -0,0 +1,23 @@
+from ._base import State
+from ._states import (
+    ConceptualPhysicalState,
+    ConceptualState,
+    EmptyState,
+    ForbiddenState,
+    InstancesConceptualPhysicalState,
+    InstancesConceptualState,
+    InstancesState,
+    PhysicalState,
+)
+
+__all__ = [
+    "ConceptualPhysicalState",
+    "ConceptualState",
+    "EmptyState",
+    "ForbiddenState",
+    "InstancesConceptualPhysicalState",
+    "InstancesConceptualState",
+    "InstancesState",
+    "PhysicalState",
+    "State",
+]

cognite/neat/_session/_state_machine/_base.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+
+
+class State(ABC):
+    def __init__(self) -> None:
+        # this will be reference to the actual store in the session
+        # used to store data models and instances, here only as a placeholder
+        self._store = None
+
+    @abstractmethod
+    def on_event(self, event: str) -> "State":
+        """
+        Handle events that are delegated to this State.
+        """
+        raise NotImplementedError("on_event() must be implemented by the subclass.")
+
+    def __repr__(self) -> str:
+        """
+        Leverages the __str__ method to describe the State.
+        """
+        return self.__str__()
+
+    def __str__(self) -> str:
+        """
+        Returns the name of the State.
+        """
+        return self.__class__.__name__

cognite/neat/_session/_state_machine/_states.py

@@ -0,0 +1,150 @@
+from ._base import State
+
+
+class EmptyState(State):
+    """
+    The initial state with empty NEAT store.
+    """
+
+    def on_event(self, event: str) -> State:
+        if event == "read_instances":
+            return InstancesState()
+        elif event == "read_conceptual":
+            return ConceptualState()
+        elif event == "read_physical":
+            return PhysicalState()
+
+        return ForbiddenState(self)
+
+
+class InstancesState(State):
+    """
+    State with instances loaded to the store.
+    """
+
+    def on_event(self, event: str) -> State:
+        # One can keep on reading instances to stay in the same state
+        if event == "read_instances":
+            return InstancesState()
+        # We either read conceptual model or infer it from instances
+        # if read conceptual, we need to make sure that conceptual model is compatible with instances
+        elif event in ["infer_conceptual", "read_conceptual"]:
+            return InstancesConceptualState()
+
+        # transforming instances keeps us in the same state
+        elif event == "transform_instances":
+            return InstancesState()
+        # we should allow writing out instances in RDF format but not to CDF
+        elif event == "write_instances":
+            return InstancesState()
+
+        # all other operations are forbidden
+        return ForbiddenState(self)
+
+
+class ConceptualState(State):
+    """
+    State with conceptual model loaded.
+    """
+
+    def on_event(self, event: str) -> State:
+        # re-reading of model means transformation of
+        # the current model has been done outside of NeatSession
+        # requires checking that the new model is compatible with the existing
+        if event == "read_conceptual":
+            return ConceptualState()
+
+        # when reading: requires linking between models
+        # when converting: links are automatically created
+        elif event == "read_physical" or event == "convert_to_physical":
+            return ConceptualPhysicalState()
+        elif event == "transform_conceptual":
+            return ConceptualState()
+        elif event == "write_conceptual":
+            return ConceptualState()
+
+        return ForbiddenState(self)
+
+
+class PhysicalState(State):
+    """
+    State with physical model loaded.
+    """
+
+    def on_event(self, event: str) -> State:
+        if event == "read_physical":
+            return PhysicalState()
+        elif event == "transform_physical":
+            return PhysicalState()
+        elif event == "write_physical":
+            return PhysicalState()
+        elif event == "convert_to_conceptual" or event == "read_conceptual":
+            return ConceptualPhysicalState()
+
+        return ForbiddenState(self)
+
+
+class InstancesConceptualState(State):
+    """
+    State with both instances and conceptual model loaded.
+    """
+
+    def on_event(self, event: str) -> State:
+        if event == "read_conceptual":
+            return InstancesConceptualState()
+        elif event == "transform_conceptual":
+            return InstancesConceptualState()
+        elif event in ["write_instances", "write_conceptual"]:
+            return InstancesConceptualState()
+        elif event in ["read_physical", "convert_to_physical"]:
+            return InstancesConceptualPhysicalState()
+
+        return ForbiddenState(self)
+
+
+class ConceptualPhysicalState(State):
+    """
+    State with both conceptual and physical models loaded.
+    """
+
+    def on_event(self, event: str) -> State:
+        if event == "read_physical":
+            return ConceptualPhysicalState()
+        elif event == "transform_physical":
+            return ConceptualPhysicalState()
+        elif event in ["write_conceptual", "write_physical"]:
+            return ConceptualPhysicalState()
+
+        return ForbiddenState(self)
+
+
+class InstancesConceptualPhysicalState(State):
+    """
+    State with instances, conceptual, and physical models loaded.
+    """
+
+    def on_event(self, event: str) -> State:
+        if event == "read_physical":
+            return InstancesConceptualPhysicalState()
+        elif event == "transform_physical":
+            return InstancesConceptualPhysicalState()
+        elif event in ["write_instances", "write_conceptual", "write_physical"]:
+            return InstancesConceptualPhysicalState()
+
+        return ForbiddenState(self)
+
+
+class ForbiddenState(State):
+    """
+    State representing forbidden transitions - returns to previous state.
+    """
+
+    def __init__(self, previous_state: State):
+        self.previous_state = previous_state
+        print(f"Forbidden action attempted. Returning to previous state: {previous_state}")
+
+    def on_event(self, event: str) -> State:
+        # only "undo" to trigger going back to previous state
+        if event.strip().lower() == "undo":
+            return self.previous_state
+        return self

cognite/neat/_utils/text.py

@@ -38,3 +38,25 @@ def humanize_collection(collection: Collection[Any], /, *, sort: bool = True, bi
         sequence = list(strings)
 
     return f"{', '.join(sequence[:-1])} {bind_word} {sequence[-1]}"
+
+
+def title_case(s: str) -> str:
+    """Convert a string to title case, handling underscores and hyphens.
+
+    Args:
+        s: The string to convert.
+    Returns:
+        The title-cased string.
+    Examples:
+        >>> title_case("hello world")
+        'Hello World'
+        >>> title_case("hello_world")
+        'Hello World'
+        >>> title_case("hello-world")
+        'Hello World'
+        >>> title_case("hello_world-and-universe")
+        'Hello World And Universe'
+        >>> title_case("HELLO WORLD")
+        'Hello World'
+    """
+    return " ".join(word.capitalize() for word in s.replace("_", " ").replace("-", " ").split())

cognite/neat/_utils/useful_types.py

@@ -1,7 +1,11 @@
 from collections.abc import Hashable
+from datetime import date, datetime, time, timedelta
 from typing import TypeAlias, TypeVar
 
 JsonVal: TypeAlias = None | str | int | float | bool | dict[str, "JsonVal"] | list["JsonVal"]
 
 
 T_ID = TypeVar("T_ID", bound=Hashable)
+
+# These are the types that openpyxl supports in cells
+CellValueType: TypeAlias = str | int | float | bool | datetime | date | time | timedelta | None

cognite/neat/_utils/validation.py

@@ -1,28 +1,71 @@
+from collections.abc import Callable, Mapping
+from typing import Literal
+
 from pydantic import ValidationError
 from pydantic_core import ErrorDetails
 
 
-def humanize_validation_error(error: ValidationError) -> list[str]:
+def as_json_path(loc: tuple[str | int, ...]) -> str:
+    """Converts a location tuple to a JSON path.
+
+    Args:
+        loc: The location tuple to convert.
+
+    Returns:
+        A JSON path string.
+    """
+    if not loc:
+        return ""
+    # +1 to convert from 0-based to 1-based indexing
+    prefix = ""
+    if isinstance(loc[0], int):
+        prefix = "item"
+
+    suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
+    return f"{prefix}{suffix}"
+
+
+def humanize_validation_error(
+    error: ValidationError,
+    parent_loc: tuple[int | str, ...] = tuple(),
+    humanize_location: Callable[[tuple[int | str, ...]], str] = as_json_path,
+    field_name: Literal["field", "column", "value"] = "field",
+    field_renaming: Mapping[str, str] | None = None,
+    missing_required_descriptor: Literal["empty", "missing"] = "missing",
+) -> list[str]:
     """Converts a ValidationError to a human-readable format.
 
     This overwrites the default error messages from Pydantic to be better suited for Toolkit users.
 
     Args:
         error: The ValidationError to convert.
-
+        parent_loc: Optional location tuple to prepend to each error location.
+            This is useful when the error is for a nested model and you want to include the location
+            of the parent model.
+        humanize_location: A function that converts a location tuple to a human-readable string.
+            The default is `as_json_path`, which converts the location to a JSON path.
+            This can for example be replaced when the location comes from an Excel table.
+        field_name: The name use for "field" in error messages. Default is "field". This can be changed to
+            "column" or "value" to better fit the context.
+        field_renaming: Optional mapping of field names to source names.
+            This is useful when the field names in the model are different from the names in the source.
+            For example, if the model field is "asset_id" but the source column is "Asset ID",
+            you can provide a mapping {"asset_id": "Asset ID"} to have the error messages use the source names.
+        missing_required_descriptor: How to describe missing required fields. Default is "missing".
+            Other option is "empty" which can be more suitable for table data.
     Returns:
         A list of human-readable error messages.
     """
     errors: list[str] = []
     item: ErrorDetails
-
+    field_renaming = field_renaming or {}
     for item in error.errors(include_input=True, include_url=False):
-        loc = item["loc"]
+        loc = (*parent_loc, *item["loc"])
         error_type = item["type"]
         if error_type == "missing":
-            msg = f"Missing required field: {loc[-1]!r}"
+            msg = f"Missing required {field_name}: {loc[-1]!r}"
         elif error_type == "extra_forbidden":
-            msg = f"Unused field: {loc[-1]!r}"
+            msg = f"Unused {field_name}: {loc[-1]!r}"
         elif error_type == "value_error":
             msg = str(item["ctx"]["error"])
         elif error_type == "literal_error":
@@ -50,32 +93,22 @@ def humanize_validation_error(error: ValidationError) -> list[str]:
             #  This is hard to read, so we simplify it to just the field name.
             loc = tuple(["dict" if isinstance(x, str) and "json-or-python" in x else x for x in loc])
 
+        error_suffix = f"{msg[:1].casefold()}{msg[1:]}"
         if len(loc) > 1 and error_type in {"extra_forbidden", "missing"}:
-            # We skip the last element as this is in the message already
-            msg = f"In {as_json_path(loc[:-1])} {msg[:1].casefold()}{msg[1:]}"
+            if missing_required_descriptor == "empty" and error_type == "missing":
+                # This is a table so we modify the error message.
+                msg = (
+                    f"In {humanize_location(loc[:-1])} the {field_name} {field_renaming.get(str(loc[-1]), loc[-1])!r} "
+                    "cannot be empty."
+                )
+            else:
+                # We skip the last element as this is in the message already
+                msg = f"In {humanize_location(loc[:-1])} {error_suffix.replace('field', field_name)}"
         elif len(loc) > 1:
-            msg = f"In {as_json_path(loc)} {msg[:1].casefold()}{msg[1:]}"
+            msg = f"In {humanize_location(loc)} {error_suffix}"
         elif len(loc) == 1 and isinstance(loc[0], str) and error_type not in {"extra_forbidden", "missing"}:
-            msg = f"In field {loc[0]} {msg[:1].casefold()}{msg[1:]}"
+            msg = f"In {field_name} {loc[0]!r}, {error_suffix}"
+        if not msg.endswith("."):
+            msg += "."
         errors.append(msg)
     return errors
-
-
-def as_json_path(loc: tuple[str | int, ...]) -> str:
-    """Converts a location tuple to a JSON path.
-
-    Args:
-        loc: The location tuple to convert.
-
-    Returns:
-        A JSON path string.
-    """
-    if not loc:
-        return ""
-    # +1 to convert from 0-based to 1-based indexing
-    prefix = ""
-    if isinstance(loc[0], int):
-        prefix = "item "
-
-    suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
-    return f"{prefix}{suffix}"

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.123.43"
+__version__ = "0.125.0"
 __engine__ = "^2.0.4"

cognite/neat/v0/core/_data_model/_constants.py

@@ -43,6 +43,7 @@ class EntityTypes(StrEnum):
     prefix = "prefix"
     space = "space"
     container_index = "container_index"
+    container_constraint = "container_constraint"
     concept_restriction = "conceptRestriction"
     value_constraint = "valueConstraint"
     cardinality_constraint = "cardinalityConstraint"

cognite/neat/v0/core/_data_model/exporters/_data_model2excel.py

@@ -32,7 +32,7 @@ from cognite.neat.v0.core._data_model.models.data_types import (
 )
 from cognite.neat.v0.core._data_model.models.physical._verified import PhysicalDataModel
 from cognite.neat.v0.core._utils.spreadsheet import (
-    find_column_with_value,
+    find_column_and_row_with_value,
     generate_data_validation,
 )
 
@@ -217,7 +217,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
                 continue
             ws = workbook[sheet]
             for col in get_internal_properties():
-                column_letter = find_column_with_value(ws, col)
+                column_letter = find_column_and_row_with_value(ws, col)[0]
                 if column_letter:
                     ws.column_dimensions[column_letter].hidden = True
 
@@ -451,7 +451,7 @@ class ExcelExporter(BaseExporter[VerifiedDataModel, Workbook]):
         workbook[sheet_name].add_data_validation(data_validators[data_validator_name])
 
         # APPLY VALIDATOR TO SPECIFIC COLUMN
-        if column_letter := find_column_with_value(workbook[sheet_name], column_name):
+        if column_letter := find_column_and_row_with_value(workbook[sheet_name], column_name)[0]:
             data_validators[data_validator_name].add(f"{column_letter}{3}:{column_letter}{3 + total_rows}")
 
     def _create_sheet_with_header(

cognite/neat/v0/core/_data_model/importers/_dms2data_model.py

@@ -54,6 +54,7 @@ from cognite.neat.v0.core._data_model.models.entities import (
     ReverseConnectionEntity,
     ViewEntity,
 )
+from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity
 from cognite.neat.v0.core._data_model.models.physical import (
     UnverifiedPhysicalContainer,
     UnverifiedPhysicalEnum,
@@ -571,17 +572,17 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
                 index.append(ContainerIndexEntity(prefix="inverted", suffix=index_name, order=order))
         return index or None
 
-    def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
+    def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[ContainerConstraintEntity] | None:
         if not isinstance(prop, dm.MappedPropertyApply):
             return None
         container = self._all_containers_by_id[prop.container]
-        unique_constraints: list[str] = []
+        unique_constraints: list[ContainerConstraintEntity] = []
         for constraint_name, constraint_obj in (container.constraints or {}).items():
             if isinstance(constraint_obj, dm.RequiresConstraint):
                 # This is handled in the .from_container method of DMSContainer
                 continue
             elif isinstance(constraint_obj, dm.UniquenessConstraint) and prop_id in constraint_obj.properties:
-                unique_constraints.append(constraint_name)
+                unique_constraints.append(ContainerConstraintEntity(prefix="uniqueness", suffix=constraint_name))
             elif isinstance(constraint_obj, dm.UniquenessConstraint):
                 # This does not apply to this property
                 continue

cognite/neat/v0/core/_data_model/importers/_spreadsheet2data_model.py

@@ -10,8 +10,10 @@ import pandas as pd
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
 from pandas import ExcelFile
+from pydantic import ValidationError
 from rdflib import Namespace, URIRef
 
+from cognite.neat.v0.core._data_model._constants import SPLIT_ON_COMMA_PATTERN
 from cognite.neat.v0.core._data_model._shared import (
     ImportedDataModel,
     T_UnverifiedDataModel,
@@ -23,6 +25,7 @@ from cognite.neat.v0.core._data_model.models import (
     SchemaCompleteness,
 )
 from cognite.neat.v0.core._data_model.models._import_contexts import SpreadsheetContext
+from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity, ContainerEntity
 from cognite.neat.v0.core._issues import IssueList, MultiValueError
 from cognite.neat.v0.core._issues.errors import (
     FileMissingRequiredFieldError,
@@ -30,7 +33,11 @@ from cognite.neat.v0.core._issues.errors import (
     FileReadError,
 )
 from cognite.neat.v0.core._issues.warnings import FileMissingRequiredFieldWarning
-from cognite.neat.v0.core._utils.spreadsheet import SpreadsheetRead, read_individual_sheet
+from cognite.neat.v0.core._utils.spreadsheet import (
+    SpreadsheetRead,
+    find_column_and_row_with_value,
+    read_individual_sheet,
+)
 from cognite.neat.v0.core._utils.text import humanize_collection
 
 from ._base import BaseImporter
@@ -306,7 +313,7 @@ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
 
         """
 
-        workbook = load_workbook(filepath)
+        workbook = load_workbook(filepath, data_only=True)
 
         if "Classes" in workbook.sheetnames:
             print(
@@ -323,13 +330,17 @@ class ExcelImporter(BaseImporter[T_UnverifiedDataModel]):
             if "Properties" in workbook.sheetnames:
                 _replace_class_with_concept_cell(workbook["Properties"])
 
-            with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
-                workbook.save(temp_file.name)
-                return Path(temp_file.name)
+        elif "Containers" in workbook.sheetnames:
+            _replace_legacy_constraint_form(workbook["Containers"])
+            _replace_legacy_constraint_form(workbook["Properties"])
 
         else:
             return filepath
 
+        with tempfile.NamedTemporaryFile(prefix="temp_neat_file", suffix=".xlsx", delete=False) as temp_file:
+            workbook.save(temp_file.name)
+            return Path(temp_file.name)
+
 
 def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
     """
@@ -342,3 +353,72 @@ def _replace_class_with_concept_cell(sheet: Worksheet) -> None:
         for cell in row:
             if cell.value == "Class":
                 cell.value = "Concept"
+
+
+def _replace_legacy_constraint_form(sheet: Worksheet) -> None:
+    """
+    Replaces the legacy form of container constraints with the new form in the given sheet.
+
+    Args:
+        sheet (Worksheet): The sheet in which to replace the old form of container constraints.
+    """
+    column, row = find_column_and_row_with_value(sheet, "Constraint", False)
+
+    if not column or not row:
+        return None
+
+    # Iterate over values in the constraint column and replace old form with new form
+    replaced: bool = False
+    for cell_row in sheet.iter_rows(min_row=row + 1, min_col=column, max_col=column):
+        cell = cell_row[0]
+        if cell.value is not None:  # Skip empty cells
+            # Container sheet update
+            if sheet.title.lower() == "containers":
+                constraints = []
+                for constraint in SPLIT_ON_COMMA_PATTERN.split(str(cell.value)):
+                    # latest format, do nothing
+                    if "container" in constraint.lower():
+                        constraints.append(constraint)
+                        continue
+
+                    try:
+                        container = ContainerEntity.load(constraint, space="default")
+                        container_str = container.external_id if container.space == "default" else str(container)
+                        constraints.append(
+                            f"requires:{container.space}_{container.external_id}(container={container_str})"
+                        )
+                        replaced = True
+                    except ValidationError:
+                        constraints.append(constraint)
+
+                cell.value = ",".join(constraints)
+
+            # Properties sheet update
+            elif sheet.title.lower() == "properties":
+                constraints = []
+                for constraint in SPLIT_ON_COMMA_PATTERN.split(str(cell.value)):
+                    try:
+                        constraint_entity = ContainerConstraintEntity.load(constraint)
+
+                        if constraint_entity.prefix in ["uniqueness", "requires"]:
+                            constraints.append(constraint)
+
+                        # Replace old format with new format
+                        else:
+                            constraints.append(f"uniqueness:{constraint}")
+                            replaced = True
+
+                    # If the constraint is not valid, we keep it as is
+                    # to be caught by validation later
+                    except ValidationError:
+                        constraints.append(constraint)
+
+                cell.value = ",".join(constraints)
+
+    if replaced:
+        print(
+            (
+                "You are using a legacy container constraints format "
+                f"in the {sheet.title} sheet. Please update to the latest format."
+            ),
+        )

cognite/neat/v0/core/_data_model/models/entities/__init__.py

@@ -7,6 +7,7 @@ from ._single_value import (
     AssetFields,
     ConceptEntity,
     ConceptualEntity,
+    ContainerConstraintEntity,
     ContainerEntity,
     ContainerIndexEntity,
     DataModelEntity,
@@ -35,6 +36,7 @@ __all__ = [
     "ConceptPropertyCardinalityConstraint",
     "ConceptPropertyValueConstraint",
     "ConceptualEntity",
+    "ContainerConstraintEntity",
     "ContainerEntity",
     "ContainerEntityList",
     "ContainerIndexEntity",

cognite/neat/v0/core/_data_model/models/entities/_single_value.py

@@ -660,3 +660,17 @@ class ContainerIndexEntity(PhysicalEntity[None]):
     @classmethod
     def from_id(cls, id: None) -> Self:
         return cls(suffix="dummy")
+
+
+class ContainerConstraintEntity(PhysicalEntity[None]):
+    type_: ClassVar[EntityTypes] = EntityTypes.container_constraint
+    prefix: _UndefinedType | Literal["uniqueness", "requires"] = Undefined  # type: ignore[assignment]
+    suffix: str
+    container: ContainerEntity | None = None
+
+    def as_id(self) -> None:
+        return None
+
+    @classmethod
+    def from_id(cls, id: None) -> Self:
+        return cls(suffix="dummy")