cognite-neat 0.123.24__py3-none-any.whl → 1.0.22__py3-none-any.whl

cognite/neat/_data_model/importers/_table_importer/data_classes.py

@@ -0,0 +1,344 @@
+from collections.abc import Mapping
+from typing import Annotated, Literal, cast, get_args
+
+from pydantic import (
+    AliasGenerator,
+    BaseModel,
+    BeforeValidator,
+    Field,
+    PlainSerializer,
+    field_validator,
+    model_validator,
+)
+from pydantic.alias_generators import to_camel
+from pydantic.fields import FieldInfo
+from traitlets import Any
+
+from cognite.neat._data_model.models.entities import ParsedEntity, parse_entities, parse_entity
+from cognite.neat._utils.text import title_case
+from cognite.neat._utils.useful_types import CellValueType
+
+# This marker is used to identify creator in the description field.
+CREATOR_MARKER = "Creator: "
+CREATOR_KEY = "creator"
+
+
+def parse_entity_str(v: str) -> ParsedEntity:
+    if isinstance(v, ParsedEntity):
+        return v
+    try:
+        return parse_entity(v)
+    except ValueError as e:
+        raise ValueError(f"Invalid entity syntax: {e}") from e
+
+
+def parse_entities_str(v: str) -> list[ParsedEntity] | None:
+    if isinstance(v, list) and all(isinstance(item, ParsedEntity) for item in v):
+        return v
+    try:
+        return parse_entities(v)
+    except ValueError as e:
+        raise ValueError(f"Invalid entity list syntax: {e}") from e
+
+
+Entity = Annotated[ParsedEntity, BeforeValidator(parse_entity_str, str), PlainSerializer(func=str)]
+EntityList = Annotated[
+    list[ParsedEntity],
+    BeforeValidator(parse_entities_str, str),
+    PlainSerializer(func=lambda v: ",".join([str(item) for item in v])),
+]
+
+
+class TableObj(
+    BaseModel,
+    extra="ignore",
+    alias_generator=AliasGenerator(
+        alias=to_camel,
+        validation_alias=title_case,
+        serialization_alias=title_case,
+    ),
+    populate_by_name=True,
+): ...
+
+
+class MetadataValue(TableObj):
+    key: str
+    value: CellValueType
+
+    @field_validator("key", mode="after")
+    def _legacy_external_id(cls, value: str) -> str:
+        return "externalId" if value.lower() == "external_id" else value
+
+
+class DMSProperty(TableObj):
+    view: Entity
+    view_property: str
+    name: str | None = None
+    description: str | None = None
+    connection: Entity | None
+    value_type: Entity
+    min_count: int | None
+    max_count: int | None
+    immutable: bool | None = None
+    default: CellValueType | None = None
+    auto_increment: bool | None = None
+    container: Entity | None = None
+    container_property: str | None = None
+    container_property_name: str | None = None
+    container_property_description: str | None = None
+    index: EntityList | None = None
+    constraint: EntityList | None = None
+
+    @field_validator("max_count", mode="before")
+    @classmethod
+    def _legacy_max_count(cls, value: Any) -> Any | None:
+        """Validates and converts the max_count field if it uses the legacy 'inf' value."""
+        if isinstance(value, str) and value.lower() == "inf":
+            return None
+        return value
+
+    @model_validator(mode="before")
+    def _legacy_cardinality(cls, data: dict[str, Any]) -> dict[str, Any]:
+        """Converts Is List to Max Count and Nullable to Min Count."""
+        if not data:
+            return data
+
+        if "Max Count" not in data and "Is List" in data:
+            is_list = data.pop("Is List")
+            if isinstance(is_list, bool):
+                data["Max Count"] = None if is_list else 1
+
+        if "Min Count" not in data and "Nullable" in data:
+            nullable = data.pop("Nullable")
+            if isinstance(nullable, bool):
+                data["Min Count"] = 0 if nullable else 1
+
+        return data
+
+    @model_validator(mode="after")
+    def _legacy_index(self) -> "DMSProperty":
+        """Converts Is List to Max Count and Nullable to Min Count."""
+        if not self.index:
+            return self
+
+        for index in self.index:
+            if not index.prefix:
+                index.prefix = "inverted" if not self.max_count or self.max_count > 1 else "btree"
+
+        return self
+
+    @model_validator(mode="after")
+    def _legacy_constraint(self) -> "DMSProperty":
+        """Converts Is List to Max Count and Nullable to Min Count."""
+        if not self.constraint:
+            return self
+
+        for constraint in self.constraint:
+            if not constraint.prefix:
+                constraint.prefix = "uniqueness"
+
+        return self
+
+
+class EntityTableFilter(BaseModel):
+    """These are special formats that Neat Table format supports for filters."""
+
+    type: Literal["hasData", "nodeType"]
+    entities: EntityList
+
+    def __str__(self) -> str:
+        entities_str = ",".join(str(entity) for entity in self.entities)
+        return f"{self.type}({entities_str})"
+
+
+class RAWFilterTableFilter(BaseModel):
+    """This is a generic filter that holds raw JSON filter."""
+
+    type: Literal["rawFilter"] = "rawFilter"
+    filter: str
+
+    def __str__(self) -> str:
+        return f"rawFilter({self.filter})"
+
+
+def _parse_table_filter(v: str) -> dict[str, str] | EntityTableFilter | RAWFilterTableFilter:
+    if isinstance(v, EntityTableFilter | RAWFilterTableFilter):
+        return v
+    filter_configs = {
+        "hasdata(": ("hasData", "entities"),
+        "nodetype(": ("nodeType", "entities"),
+        "rawfilter(": ("rawFilter", "filter"),
+    }
+    v_lowered = v.casefold()
+    for prefix, (filter_type, field_name) in filter_configs.items():
+        if v_lowered.startswith(prefix) and v_lowered.endswith(")"):
+            return {"type": filter_type, field_name: v[len(prefix) : -1]}
+    # Fallback to raw filter with the whole string
+    return {"type": "rawFilter", "filter": v}
+
+
+TableViewFilter = Annotated[
+    EntityTableFilter | RAWFilterTableFilter,
+    Field(discriminator="type"),
+    BeforeValidator(_parse_table_filter, str),
+    PlainSerializer(func=str),
+]
+
+
+class DMSView(TableObj):
+    view: Entity
+    name: str | None = None
+    description: str | None = None
+    implements: EntityList | None = None
+    filter: TableViewFilter | None = None
+    in_model: bool | None = Field(None, exclude=True, description="Legacy column")
+
+
+class DMSContainer(TableObj):
+    container: Entity
+    name: str | None = None
+    description: str | None = None
+    constraint: EntityList | None = None
+    used_for: str | None = None
+
+    @model_validator(mode="after")
+    def _legacy_constraint(self) -> "DMSContainer":
+        """Converts legacy constraint formats to the current standard."""
+        if not self.constraint:
+            return self
+
+        for constraint in self.constraint:
+            # Skip if already in correct format or being wrong but not legacy
+            if constraint.prefix == "requires" or constraint.properties:
+                continue
+
+            # This part handles legacy constraints
+            if not constraint.prefix:
+                constraint.prefix = "requires"
+                constraint.properties = {"require": constraint.suffix}
+
+            else:
+                container_space = constraint.prefix
+                container_external_id = constraint.suffix
+                constraint.prefix = "requires"
+                constraint.suffix = f"{container_space}_{container_external_id}"
+                constraint.properties = {"require": f"{container_space}:{container_external_id}"}
+
+        return self
+
+
+class DMSEnum(TableObj):
+    collection: str
+    value: str
+    name: str | None = None
+    description: str | None = None
+
+
+class DMSNode(TableObj):
+    node: Entity
+
+
+class TableDMS(TableObj):
+    metadata: list[MetadataValue]
+    properties: list[DMSProperty]
+    views: list[DMSView]
+    containers: list[DMSContainer] = Field(default_factory=list)
+    enum: list[DMSEnum] = Field(default_factory=list)
+    nodes: list[DMSNode] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    def _title_case_keys(
+        cls, data: dict[str, list[dict[str, CellValueType]]]
+    ) -> dict[str, list[dict[str, CellValueType]]]:
+        if isinstance(data, dict):
+            # We are case-insensitive on the table names.
+            return {title_case(k): v for k, v in data.items()}
+        return data
+
+    @model_validator(mode="after")
+    def _drop_in_model_false_views_definitions(self) -> "TableDMS":
+        """These method is used to drop definition of legacy views which have In Model column set to False
+        We need to drop these views from Views sheet and also drop their properties from Properties sheet.
+        """
+
+        views_to_drop: set[Entity] = set()
+        for view in self.views:
+            if isinstance(view.in_model, bool) and not view.in_model:
+                views_to_drop.add(view.view)
+
+        if not views_to_drop:
+            return self
+
+        print(
+            "You are using legacy `In Model` column which is no longer in use!"
+            f"\nTotal of {len(views_to_drop)} views has `In Model` set to False."
+            f"\nThese views and their property definitions will be dropped from the session!"
+        )
+
+        self.views = [view for view in self.views if view.view not in views_to_drop]
+        self.properties = [prop for prop in self.properties if prop.view not in views_to_drop]
+
+        return self
+
+    @classmethod
+    def get_sheet_columns(
+        cls, sheet_id: str, sheet: FieldInfo | None = None, *, column_type: Literal["all", "required"] = "required"
+    ) -> list[str]:
+        if sheet_id not in cls.model_fields.keys():
+            raise KeyError(f"Invalid field id: {sheet_id}")
+        if sheet is None:
+            sheet = cls.model_fields[sheet_id]
+        return [
+            # We know all fields has validation_alias because of the alias_generator in TableDMS
+            cast(str, sheet_field.validation_alias)
+            # All the fields in the sheet's model are lists.
+            for sheet_field in get_args(sheet.annotation)[0].model_fields.values()
+            if sheet_field.is_required() or column_type == "all"
+        ]
+
+    @classmethod
+    def get_sheet_column_by_name(
+        cls, sheet_name: str, *, column_type: Literal["all", "required"] = "required"
+    ) -> list[str]:
+        for field_id, field_ in cls.model_fields.items():
+            if cast(str, field_.validation_alias) == sheet_name:
+                return cls.get_sheet_columns(field_id, field_, column_type=column_type)
+        raise KeyError(f"Invalid field alias: {sheet_name}")
+
+    @classmethod
+    def required_sheets(cls) -> set[str]:
+        return {cast(str, field_.validation_alias) for field_ in cls.model_fields.values() if field_.is_required()}
+
+
+DMS_API_MAPPING: Mapping[str, Mapping[str, str]] = {
+    "Views": {
+        "space": "View",
+        "externalId": "View",
+        "version": "View",
+        **{
+            cast(str, field_.alias): cast(str, field_.validation_alias)
+            for field_id, field_ in DMSView.model_fields.items()
+            if field_id != "View"
+        },
+    },
+    "Containers": {
+        "space": "Container",
+        "externalId": "Container",
+        **{
+            cast(str, field_.alias): cast(str, field_.validation_alias)
+            for field_id, field_ in DMSContainer.model_fields.items()
+            if field_id != "Container"
+        },
+    },
+    "Properties": {
+        "space": "View",
+        "externalId": "View",
+        "property": "ViewProperty",
+        "type": "Value Type",
+        **{
+            cast(str, field_.alias): cast(str, field_.validation_alias)
+            for field_id, field_ in DMSProperty.model_fields.items()
+            if field_id not in ("View", "ViewProperty")
+        },
+    },
+}

cognite/neat/_data_model/importers/_table_importer/importer.py

@@ -0,0 +1,192 @@
+import warnings
+from collections.abc import Mapping
+from pathlib import Path
+from typing import ClassVar, cast
+
+import yaml
+from openpyxl import load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+from pydantic import ValidationError
+
+from cognite.neat._data_model.importers._base import DMSImporter
+from cognite.neat._data_model.models.dms import (
+    RequestSchema,
+)
+from cognite.neat._exceptions import DataModelImportException
+from cognite.neat._issues import ModelSyntaxError
+from cognite.neat._utils.text import humanize_collection
+from cognite.neat._utils.useful_types import CellValueType, DataModelTableType
+from cognite.neat._utils.validation import ValidationContext, as_json_path, humanize_validation_error
+
+from .data_classes import MetadataValue, TableDMS
+from .reader import DMSTableReader
+from .source import SpreadsheetReadContext, TableSource
+
+
+class DMSTableImporter(DMSImporter):
+    """Imports DMS from a table structure.
+
+    The tables can are expected to be a dictionary where the keys are the table names and the values
+    are lists of dictionaries representing the rows in the table.
+    """
+
+    # We can safely cast as we know the validation_alias is always set to a str.
+    REQUIRED_SHEETS = tuple(
+        cast(str, field_.validation_alias) for field_ in TableDMS.model_fields.values() if field_.is_required()
+    )
+    REQUIRED_SHEET_MESSAGES: ClassVar[Mapping[str, str]] = {
+        f"Missing required column: {sheet!r}": f"Missing required sheet: {sheet!r}" for sheet in REQUIRED_SHEETS
+    }
+    MetadataSheet = cast(str, TableDMS.model_fields["metadata"].validation_alias)
+
+    def __init__(self, tables: DataModelTableType, source: TableSource | None = None) -> None:
+        self._table = tables
+        self._source = source or TableSource("Unknown")
+
+    def to_data_model(self) -> RequestSchema:
+        tables = self._read_tables()
+
+        space, version = self._read_defaults(tables.metadata)
+        reader = DMSTableReader(space, version, self._source)
+        return reader.read_tables(tables)
+
+    @classmethod
+    def from_yaml(cls, yaml_file: Path) -> "DMSTableImporter":
+        """Create a DMSTableImporter from a YAML file."""
+        source = cls._display_name(yaml_file)
+        return cls(yaml.safe_load(yaml_file.read_text()), TableSource(source.as_posix()))
+
+    @classmethod
+    def from_json(cls, json_file: Path) -> "DMSTableImporter":
+        """Create a DMSTableImporter from a JSON file."""
+        return cls.from_yaml(json_file)
+
+    @classmethod
+    def from_excel(cls, excel_file: Path) -> "DMSTableImporter":
+        """Create a DMSTableImporter from an Excel file."""
+        tables: DataModelTableType = {}
+        source = TableSource(cls._display_name(excel_file).as_posix())
+        workbook = load_workbook(excel_file, read_only=True, data_only=True, rich_text=False)
+        try:
+            for column_id, column_info in TableDMS.model_fields.items():
+                sheet_name = cast(str, column_info.validation_alias)
+                if sheet_name not in workbook.sheetnames:
+                    continue
+                required_columns = TableDMS.get_sheet_columns(column_id, column_info, column_type="required")
+                sheet = workbook[sheet_name]
+                context = SpreadsheetReadContext()
+                table_rows = cls._read_rows(sheet, required_columns, context)
+                tables[sheet_name] = table_rows
+                source.table_read[sheet_name] = context
+            return cls(tables, source)
+        finally:
+            workbook.close()
+
+    def _read_tables(self) -> TableDMS:
+        try:
+            # Check tables, columns, data type and entity syntax.
+            table = TableDMS.model_validate(self._table)
+        except ValidationError as e:
+            errors = self._create_error_messages(e)
+            raise DataModelImportException(errors) from None
+        return table
+
+    def _create_error_messages(self, error: ValidationError) -> list[ModelSyntaxError]:
+        errors: list[ModelSyntaxError] = []
+        context = ValidationContext(
+            humanize_location=self._location,
+            field_name="column",
+            missing_required_descriptor="missing",
+        )
+        seen: set[str] = set()
+
+        for error_details in error.errors(include_input=True, include_url=False):
+            message = humanize_validation_error(error_details, context)
+            # Replace messages about missing required columns with missing required sheets.
+            message = self.REQUIRED_SHEET_MESSAGES.get(message, message)
+            if message in seen:
+                # We treat all rows as the same, so we get duplicated errors for each row.
+                continue
+
+            seen.add(message)
+            errors.append(ModelSyntaxError(message=message))
+
+        return errors
+
+    def _location(self, loc: tuple[str | int, ...]) -> str:
+        if isinstance(loc[0], str) and len(loc) == 2:  # Sheet + row.
+            # We skip the row as we treat all rows as the same. For example, if a required column is missing in one
+            # row, it is missing in all rows.
+            return f"{loc[0]} sheet"
+        elif len(loc) == 3 and isinstance(loc[0], str) and isinstance(loc[1], int) and isinstance(loc[2], str):
+            # This means there is something wrong in a specific cell.
+
+            sheet = loc[0]
+            row = loc[1]
+            if self._source and sheet in self._source.table_read:
+                context = self._source.table_read[sheet]
+                row = context.adjusted_row_number(row) - 1
+
+            return f"{sheet} sheet row {row + 1} column {loc[2]!r}"
+        # This should be unreachable as the TableDMS model only has 2 levels.
+        return as_json_path(loc)
+
+    @staticmethod
+    def _read_defaults(metadata: list[MetadataValue]) -> tuple[str, str]:
+        """Reads the space and version from the metadata table."""
+        default_space: str | None = None
+        default_version: str | None = None
+        missing = {"space", "version"}
+        for meta in metadata:
+            if meta.key == "space":
+                default_space = str(meta.value)
+                missing.remove("space")
+            elif meta.key == "version":
+                default_version = str(meta.value)
+                missing.remove("version")
+        if missing:
+            error = ModelSyntaxError(message=f"In Metadata missing required values: {humanize_collection(missing)}")
+            # If space or version is missing, we cannot continue parsing the model as these are used as defaults.
+            raise DataModelImportException([error]) from None
+        return str(default_space), str(default_version)
+
+    @classmethod
+    def _display_name(cls, filepath: Path) -> Path:
+        """Get a display-friendly version of the file path."""
+        cwd = Path.cwd()
+        source = filepath
+        if filepath.is_relative_to(cwd):
+            source = filepath.relative_to(cwd)
+        return source
+
+    @classmethod
+    def _read_rows(
+        cls, sheet: Worksheet, required_columns: list[str], context: SpreadsheetReadContext
+    ) -> list[dict[str, CellValueType]]:
+        table_rows: list[dict[str, CellValueType]] = []
+        # Metadata sheet is just a key-value pair of the first two columns.
+        # For other sheets, we need to find the column header row first.
+        columns: list[str] = [] if sheet.title != cls.MetadataSheet else required_columns
+        # Ignore warnings about Data Validation extension not being supported in read-only mode.
+        warnings.filterwarnings("ignore", category=UserWarning, message="Data Validation extension is not supported*")
+        # The .iter_rows with values_only=True raises the warning `UserWarning: Data Validation extension is
+        # not supported and will be removed`
+        # which confuses the user and we do not depend on data validation here, so we suppress it.
+        for row_no, row in enumerate(sheet.iter_rows(values_only=True)):
+            if columns:
+                # We have found the column header row, read the data rows.
+                if all(cell is None for cell in row):
+                    context.empty_rows.append(row_no)
+                else:
+                    record = dict(zip(columns, row, strict=False))
+                    # MyPy complains as it thinks DataTableFormula | ArrayFormula could be cell values,
+                    # but as we used values_only=True, this is not the case.
+                    table_rows.append(record)  # type: ignore[arg-type]
+            else:
+                # Look for the column header row.
+                row_values = [str(cell) for cell in row]
+                if set(row_values).intersection(required_columns):
+                    columns = row_values
+                    context.header_row = row_no
+
+        return table_rows