cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/utils/dtype_conversion.py

@@ -2,14 +2,31 @@ import ctypes
 import json
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
-from typing import ClassVar, cast
-
-from cognite.client.data_classes.data_modeling.data_types import Enum, ListablePropertyType, PropertyType
+from datetime import datetime
+from typing import ClassVar, Literal, overload
+
+from cognite.client.data_classes import Label, LabelDefinition
+from cognite.client.data_classes.data_modeling import ContainerId
+from cognite.client.data_classes.data_modeling.data_types import (
+    DirectRelationReference,
+    Enum,
+    ListablePropertyType,
+    PropertyType,
+)
 from cognite.client.data_classes.data_modeling.instances import PropertyValueWrite
 from cognite.client.utils import ms_to_datetime
 from dateutil import parser
 
+from cognite_toolkit._cdf_tk.constants import CDF_UNIT_SPACE
 from cognite_toolkit._cdf_tk.exceptions import ToolkitNotSupported
+from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
+from cognite_toolkit._cdf_tk.utils.useful_types import (
+    AssetCentricType,
+    AssetCentricTypeExtended,
+    DataType,
+    JsonVal,
+    PythonTypes,
+)
 
 from .collection import humanize_collection
 
@@ -19,8 +36,32 @@ INT64_MIN = -9_223_372_036_854_775_808
 INT64_MAX = 9_223_372_036_854_775_807
 
 
+def asset_centric_convert_to_primary_property(
+    value: str | int | float | bool | dict | list | None,
+    type_: PropertyType,
+    nullable: bool,
+    destination_container_property: tuple[ContainerId, str],
+    source_property: tuple[AssetCentricTypeExtended, str],
+    direct_relation_lookup: Mapping[str | int, DirectRelationReference] | None = None,
+) -> PropertyValueWrite:
+    if (source_property, destination_container_property) in SPECIAL_CONVERTER_BY_SOURCE_DESTINATION:
+        converter_cls = SPECIAL_CONVERTER_BY_SOURCE_DESTINATION[(source_property, destination_container_property)]
+        if issubclass(converter_cls, _SourceConverter):
+            if direct_relation_lookup is None:
+                raise TypeError("Direct relation lookup must be provided for DirectRelationReference conversion")
+            return converter_cls(direct_relation_lookup=direct_relation_lookup).convert(value)
+        else:
+            return converter_cls().convert(value)
+    else:
+        # Fallback to the standard conversion
+        return convert_to_primary_property(value, type_, nullable, direct_relation_lookup=direct_relation_lookup)
+
+
 def convert_to_primary_property(
-    value: str | int | float | bool | dict | list | None, type_: PropertyType, nullable: bool
+    value: str | int | float | bool | dict | list | None,
+    type_: PropertyType,
+    nullable: bool,
+    direct_relation_lookup: Mapping[str | int, DirectRelationReference] | None = None,
 ) -> PropertyValueWrite:
     """Convert a string value to the appropriate type based on the provided property type.
 
@@ -28,6 +69,8 @@ def convert_to_primary_property(
         value (str | int | float | bool): The value to convert.
         type_ (PropertyType): The type of the property to convert to.
         nullable (bool): Whether the property can be null.
+        direct_relation_lookup (Mapping[str | int, DirectRelationReference] | None): A mapping from external/internal
+            IDs to DirectRelationReference objects,
 
     Returns:
         PropertyValueWrite: The converted value as a PropertyValue, or None if is_nullable is True and the value is empty.
@@ -37,7 +80,17 @@ def convert_to_primary_property(
         converter_cls = CONVERTER_BY_DTYPE[dtype]
     else:
         raise TypeError(f"Unsupported property type {dtype}")
-    converter = converter_cls(type_, nullable)
+    if issubclass(converter_cls, _EnumConverter) and isinstance(type_, Enum):
+        converter: _ValueConverter = _EnumConverter(type_=type_, nullable=nullable)
+    elif issubclass(converter_cls, _EnumConverter):
+        raise TypeError(f"Unsupported property type {type_} for enum conversion")
+    elif issubclass(converter_cls, _DirectRelationshipConverter):
+        if direct_relation_lookup is None:
+            raise TypeError("Direct relation lookup must be provided for DirectRelationReference conversion")
+        converter = _DirectRelationshipConverter(direct_relation_lookup=direct_relation_lookup)
+    else:
+        converter = converter_cls(nullable)
+
     if isinstance(type_, ListablePropertyType) and type_.is_list:
         values = _as_list(value)
         output: list[PropertyValueWrite] = []
@@ -51,6 +104,105 @@ def convert_to_primary_property(
         return converter.convert(value)
 
 
+def convert_str_to_data_type(
+    value: str | None,
+    type_: DataType,
+    nullable: bool = True,
+    is_array: bool = False,
+) -> PythonTypes | None:
+    """Convert a string value to the appropriate data type based on the provided type.
+
+    Args:
+        value: The value to convert, which can be a string or None.
+        type_: The target data type.
+        nullable: Whether data type can be null.
+        is_array: Whether the data type is an array.
+
+    Returns:
+        The converted value as a string, int, float, bool, datetime, date, dict, list, or None.
+
+    """
+    if type_ not in DATATYPE_CONVERTER_BY_DATA_TYPE:
+        raise ValueError(
+            f"Unsupported data type {type_}. Available types: {humanize_collection(DATATYPE_CONVERTER_BY_DATA_TYPE.keys())}."
+        )
+    converter_cls = DATATYPE_CONVERTER_BY_DATA_TYPE[type_]
+    converter = converter_cls(nullable)
+    if is_array:
+        values = _as_list(value)
+        output: list[PythonTypes] = []
+        for item in values:
+            converted = converter.convert(item)  # type: ignore[arg-type]
+            if converted is not None:
+                output.append(converted)  # type: ignore[arg-type]
+        # MyPy gets confused by the SequenceNotStr used in the PropertyValueWrite
+        return output
+    else:
+        return converter.convert(value)  # type: ignore[return-value]
+
+
+@overload
+def infer_data_type_from_value(value: str, dtype: Literal["Json"]) -> tuple[DataType, JsonVal]: ...
+
+
+@overload
+def infer_data_type_from_value(value: str, dtype: Literal["Python"]) -> tuple[DataType, PythonTypes]: ...
+
+
+def infer_data_type_from_value(value: str, dtype: Literal["Json", "Python"]) -> tuple[DataType, JsonVal | PythonTypes]:
+    """Infer the data type from a given value.
+
+    Args:
+        value: The value to infer the data type from, which can be a string.
+        dtype: The data type to infer. Can be "JsonVal" or "Python". Python type will infer datetime and date types
+            as well.
+
+    Returns:
+        A tuple containing the inferred data type and the converted value.
+
+    """
+    converter_classes = (
+        _Int64Converter,
+        _Float64Converter,
+        _TimestampConverter,
+        _BooleanConverter,
+        _JsonConverter,
+        _TextConverter,
+    )
+    converters_to_use = (
+        tuple(converter_cls for converter_cls in converter_classes if converter_cls is not _TimestampConverter)
+        if dtype == "Json"
+        else converter_classes
+    )
+    for converter_cls in converters_to_use:
+        # MyPy thinks that converter_cls can be abstract, but it is not
+        converter = converter_cls(nullable=False)
+        try:
+            converted_value = converter.convert(value)
+        except ValueError:
+            continue
+
+        if (
+            converter_cls is _TimestampConverter
+            and isinstance(converted_value, datetime)
+            and _is_midnight_and_naive(converted_value)
+        ):
+            # If the converted value is a datetime with no time component, return it as a date
+            return _DateConverter.schema_type, converted_value.date()
+        else:
+            return converter_cls.schema_type, converted_value  # type: ignore[return-value]
+
+    raise ValueError(
+        f"Failed to infer data type from value: {value!r}. Supported types are: "
+        f"{humanize_collection(DATATYPE_CONVERTER_BY_DATA_TYPE.keys())}."
+    )
+
+
+def _is_midnight_and_naive(dt: datetime) -> bool:
+    """Checks if a datetime object is at midnight and is naive (has no timezone)."""
+    return not (dt.hour or dt.minute or dt.second or dt.microsecond or dt.tzinfo)
+
+
 def _as_list(value: str | int | float | bool | dict[str, object] | list[object] | None) -> list[object]:
     """Convert a value to a list, ensuring that it is iterable."""
     if value is None:
@@ -80,10 +232,10 @@ class _Converter(ABC):
 
 class _ValueConverter(_Converter, ABC):
     type_str: ClassVar[str]
+    schema_type: ClassVar[DataType | None] = None
     _handles_list: ClassVar[bool] = False
 
-    def __init__(self, type_: PropertyType, nullable: bool):
-        self.type = type_
+    def __init__(self, nullable: bool) -> None:
         self.nullable = nullable
 
     def convert(self, value: str | int | float | bool | dict | list | None) -> PropertyValueWrite:
@@ -102,8 +254,103 @@ class _ValueConverter(_Converter, ABC):
         raise NotImplementedError("This method should be implemented by subclasses.")
 
 
+class _SpecialCaseConverter(_Converter, ABC):
+    """Abstract base class for converters handling special cases."""
+
+    source_property: ClassVar[tuple[AssetCentricType, str]]
+    destination_container_property: ClassVar[tuple[ContainerId, str]]
+
+
+class _TimeSeriesTypeConverter(_SpecialCaseConverter):
+    source_property = ("timeseries", "isString")
+    destination_container_property = (ContainerId("cdf_cdm", "CogniteTimeSeries"), "type")
+
+    def convert(self, value: str | int | float | bool | dict | list | None) -> str:
+        if isinstance(value, bool):
+            return "string" if value else "numeric"
+        raise ValueError(f"Cannot convert {value} to TimeSeries type. Expected a boolean value.")
+
+
+class _TimeSeriesUnitConverter(_SpecialCaseConverter):
+    source_property = ("timeseries", "unitExternalId")
+    destination_container_property = (ContainerId("cdf_cdm", "CogniteTimeSeries"), "unit")
+
+    def convert(self, value: str | int | float | bool | dict | list | None) -> dict[str, str] | None:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            return {"space": CDF_UNIT_SPACE, "externalId": value}
+        raise ValueError(f"Cannot convert {value!r} to TimeSeries unit. Expected a string representing the externalId.")
+
+
+class _LabelConverter(_SpecialCaseConverter, ABC):
+    destination_container_property = (ContainerId("cdf_cdm", "CogniteDescribable"), "tags")
+
+    def convert(self, value: str | int | float | bool | dict | list | None) -> list[str]:
+        if not isinstance(value, list):
+            raise ValueError(
+                f"Cannot convert {value} to labels. Expected a list of Labels, objects, or LabelDefinitions."
+            )
+        tags: list[str] = []
+        for item in value:
+            match item:
+                case str() as tag:
+                    tags.append(tag)
+                # Matches a dict with the key "externalId" whose value is a string
+                case {"externalId": str() as tag}:
+                    tags.append(tag)
+                # Matches Label or LabelDefinition objects with a truthy external_id
+                case Label(external_id=tag) | LabelDefinition(external_id=tag) if tag:
+                    tags.append(tag)
+                case _:
+                    raise ValueError(
+                        f"Invalid label item: {item}. Expected a string, dict with 'externalId', or Label/LabelDefinition."
+                    )
+        return tags
+
+
+class _AssetLabelConverter(_LabelConverter):
+    source_property = ("asset", "labels")
+
+
+class _FileLabelConverter(_LabelConverter):
+    source_property = ("file", "labels")
+
+
+class _SourceConverter(_SpecialCaseConverter, ABC):
+    destination_container_property = (ContainerId("cdf_cdm", "CogniteSourceable"), "source")
+
+    def __init__(self, direct_relation_lookup: Mapping[str | int, DirectRelationReference]) -> None:
+        self.direct_relation_lookup = direct_relation_lookup
+
+    def convert(self, value: str | int | float | bool | dict | list | None) -> DirectRelationReference:
+        if isinstance(value, str | int):
+            if value in self.direct_relation_lookup:
+                return self.direct_relation_lookup[value]
+            elif isinstance(value, str) and value.casefold() in self.direct_relation_lookup:
+                # The aggregate endpoint used to create source system lowercases all sources, thus
+                # we do a case-insensitive lookup as a fallback.
+                return self.direct_relation_lookup[value.casefold()]
+        raise ValueError(
+            f"Cannot convert {value!r} to DirectRelationReference. Invalid data type or missing in lookup."
+        )
+
+
+class _AssetSourceConverter(_SourceConverter):
+    source_property = ("asset", "source")
+
+
+class _FileSourceConverter(_SourceConverter):
+    source_property = ("file", "source")
+
+
+class _EventSourceConverter(_SourceConverter):
+    source_property = ("event", "source")
+
+
 class _TextConverter(_ValueConverter):
     type_str = "text"
+    schema_type = "string"
 
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
         return str(value)
@@ -111,6 +358,7 @@ class _TextConverter(_ValueConverter):
 
 class _BooleanConverter(_ValueConverter):
     type_str = "boolean"
+    schema_type = "boolean"
 
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
         if isinstance(value, bool | int | float):
@@ -143,6 +391,7 @@ class _Int32Converter(_ValueConverter):
 
 class _Int64Converter(_ValueConverter):
     type_str = "int64"
+    schema_type = "integer"
 
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
         if isinstance(value, int):
@@ -183,6 +432,7 @@ class _Float32Converter(_ValueConverter):
 
 class _Float64Converter(_ValueConverter):
     type_str = "float64"
+    schema_type = "float"
 
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
         if isinstance(value, float | int):
@@ -201,6 +451,7 @@ class _Float64Converter(_ValueConverter):
 
 class _JsonConverter(_ValueConverter):
     type_str = "json"
+    schema_type = "json"
     _handles_list = True
 
     def _convert(self, value: str | int | float | bool | dict[str, object] | list) -> PropertyValueWrite:
@@ -211,7 +462,7 @@ class _JsonConverter(_ValueConverter):
                 raise ValueError(
                     f"JSON keys must be strings. Found non-string keys: {humanize_collection(non_string_keys)}"
                 )
-            return value  # type: ignore[return-value]
+            return value
         elif isinstance(value, list):
             if not all(isinstance(item, str | int | float | bool | dict | list) for item in value):
                 raise ValueError("All items in the list must be of type str, int, float, bool, dict, or list.")
@@ -226,8 +477,9 @@ class _JsonConverter(_ValueConverter):
 
 class _TimestampConverter(_ValueConverter):
     type_str = "timestamp"
+    schema_type = "timestamp"
 
-    def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
+    def _convert(self, value: str | int | float | bool | dict) -> datetime:
         if isinstance(value, int | float):
             try:
                 return ms_to_datetime(value)
@@ -241,8 +493,26 @@ class _TimestampConverter(_ValueConverter):
         raise ValueError(f"Cannot convert {value} to timestamp.")
 
 
+class _EpochConverter(_ValueConverter):
+    type_str = "epoch"
+    schema_type = "epoch"
+
+    def _convert(self, value: str | int | float | bool | dict) -> int:
+        if isinstance(value, int | float):
+            return int(value)
+        elif isinstance(value, str):
+            try:
+                dt = parser.isoparse(value)
+                epoch = int(dt.timestamp() * 1000)
+                return epoch
+            except ValueError as e:
+                raise ValueError(f"Cannot convert {value} to epoch timestamp: {e}") from e
+        raise ValueError(f"Cannot convert {value} to epoch timestamp.")
+
+
 class _DateConverter(_ValueConverter):
     type_str = "date"
+    schema_type = "date"
 
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
         if isinstance(value, str):
@@ -256,22 +526,32 @@ class _DateConverter(_ValueConverter):
 class _EnumConverter(_ValueConverter):
     type_str = "enum"
 
+    def __init__(self, type_: Enum, nullable: bool) -> None:
+        super().__init__(nullable)
+        self.available_types = {enum_value.casefold(): enum_value for enum_value in type_.values.keys()}
+
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
-        type_ = cast(Enum, self.type)
-        available_types = {enum_value.casefold(): enum_value for enum_value in type_.values.keys()}
         value = str(value).casefold()
-        if value in available_types:
-            return available_types[value]
+        if value in self.available_types:
+            return self.available_types[value]
         raise ValueError(
-            f"Value {value!r} is not a valid enum value. Available values: {humanize_collection(available_types.values())}"
+            f"Value {value!r} is not a valid enum value. Available values: {humanize_collection(self.available_types.values())}"
         )
 
 
 class _DirectRelationshipConverter(_ValueConverter):
     type_str = "direct"
 
+    def __init__(self, direct_relation_lookup: Mapping[str | int, DirectRelationReference]) -> None:
+        super().__init__(nullable=True)
+        self.direct_relation_lookup = direct_relation_lookup
+
     def _convert(self, value: str | int | float | bool | dict) -> PropertyValueWrite:
-        raise ToolkitNotSupported("Direct relationship conversion is not supported.")
+        if isinstance(value, str | int) and value in self.direct_relation_lookup:
+            return self.direct_relation_lookup[value]
+        raise ValueError(
+            f"Cannot convert {value!r} to DirectRelationReference. Invalid data type or missing in lookup."
+        )
 
 
 class _TimeSeriesReferenceConverter(_ValueConverter):
@@ -299,3 +579,15 @@ CONVERTER_BY_DTYPE: Mapping[str, type[_ValueConverter]] = {
     cls_.type_str: cls_  # type: ignore[type-abstract]
     for cls_ in _ValueConverter.__subclasses__()
 }
+SPECIAL_CONVERTER_BY_SOURCE_DESTINATION: Mapping[
+    tuple[tuple[AssetCentricTypeExtended, str], tuple[ContainerId, str]],
+    type[_SpecialCaseConverter],
+] = {
+    (subclass.source_property, subclass.destination_container_property): subclass
+    for subclass in get_concrete_subclasses(_SpecialCaseConverter)  # type: ignore[type-abstract]
+}
+DATATYPE_CONVERTER_BY_DATA_TYPE: Mapping[DataType, type[_ValueConverter]] = {
+    cls_.schema_type: cls_
+    for cls_ in get_concrete_subclasses(_ValueConverter)  # type: ignore[type-abstract]
+    if cls_.schema_type is not None
+}

cognite_toolkit/_cdf_tk/utils/file.py

@@ -14,6 +14,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Literal, TypeVar, overload
+from zipfile import ZipFile
 
 import pandas as pd
 import yaml
@@ -144,6 +145,28 @@ _TOTAL_ELAPSED_TIME = 0.0
 _HAS_HINTED = False
 
 
+def keyvault_constructor(loader: yaml.Loader, node: yaml.Node) -> Any:
+    """Constructor to preserve the !keyvault tag along with the value."""
+    if isinstance(node, yaml.ScalarNode):
+        value = loader.construct_scalar(node)
+        return f"!keyvault {value}"
+    elif isinstance(node, yaml.MappingNode):
+        return loader.construct_mapping(node)
+    elif isinstance(node, yaml.SequenceNode):
+        return loader.construct_sequence(node)
+    else:
+        # This should never happen as YAML only has these three node types
+        # But we need this for type checking completeness
+        raise ValueError(f"Unexpected node type: {type(node)}")
+
+
+yaml.SafeLoader.add_constructor("!keyvault", keyvault_constructor)
+
+# Register for the fast CSafeLoader (used directly when libyaml is available)
+if hasattr(yaml, "CSafeLoader"):
+    yaml.CSafeLoader.add_constructor("!keyvault", keyvault_constructor)
+
+
 def read_yaml_content(content: str) -> dict[str, Any] | list[dict[str, Any]]:
     """Read a YAML string and return a dictionary
 
@@ -171,8 +194,8 @@ def read_yaml_content(content: str) -> dict[str, Any] | list[dict[str, Any]]:
 _ILLEGAL_CHARACTERS = re.compile(r"[<>:\"/\\|?*\s]")
 
 
-def to_directory_compatible(text: str) -> str:
-    """Convert a string to be compatible with directory names on all platforms"""
+def sanitize_filename(text: str) -> str:
+    """Convert a string to be a valid filename by replacing illegal characters with underscores."""
     cleaned = _ILLEGAL_CHARACTERS.sub("_", text)
     # Replace multiple underscores with a single one
     return re.sub(r"_+", "_", cleaned)
@@ -425,3 +448,33 @@ def get_table_columns(table: Path) -> list[str]:
         return pd.read_parquet(table).columns.tolist()
     else:
         raise ToolkitValueError(f"The file {table.name} is not a supported table format (csv, parquet)")
+
+
+@contextmanager
+def create_temporary_zip(directory: Path, zipname: str) -> typing.Generator[Path, None, None]:
+    """
+    Create a temporary zip file from a directory.
+
+    Args:
+        directory: The directory to zip.
+        zipname: The name of the zip file.
+
+    Returns:
+        A context manager that yields the path to the zip file.
+
+    """
+    with tempfile.TemporaryDirectory() as tmpdir:
+        zip_path = Path(tmpdir, zipname)
+        with ZipFile(zip_path, "w", strict_timestamps=False) as zf:
+            for root, _, files in os.walk(directory):
+                root_path = Path(root)
+                arc_root = root_path.relative_to(directory)
+
+                # Add directory entry to preserve directory structure, including empty ones.
+                zf.write(root_path, arcname=str(arc_root))
+
+                for filename in files:
+                    file_path = root_path / filename
+                    arcname = file_path.relative_to(directory)
+                    zf.write(file_path, arcname)
+        yield zip_path

cognite_toolkit/_cdf_tk/utils/fileio/__init__.py

@@ -0,0 +1,61 @@
+from ._base import CellValue, Chunk, DataType, PrimaryCellValue, SchemaColumn
+from ._compression import (
+    COMPRESSION_BY_NAME,
+    COMPRESSION_BY_SUFFIX,
+    Compression,
+    GzipCompression,
+    Uncompressed,
+)
+from ._readers import (
+    FILE_READ_CLS_BY_FORMAT,
+    TABLE_READ_CLS_BY_FORMAT,
+    CSVReader,
+    FailedParsing,
+    FileReader,
+    MultiFileReader,
+    NDJsonReader,
+    ParquetReader,
+    YAMLReader,
+    YMLReader,
+)
+from ._writers import (
+    FILE_WRITE_CLS_BY_FORMAT,
+    TABLE_WRITE_CLS_BY_FORMAT,
+    CSVWriter,
+    FileWriter,
+    NDJsonWriter,
+    ParquetWriter,
+    YAMLWriter,
+    YMLWriter,
+)
+
+__all__ = [
+    "COMPRESSION_BY_NAME",
+    "COMPRESSION_BY_SUFFIX",
+    "FILE_READ_CLS_BY_FORMAT",
+    "FILE_WRITE_CLS_BY_FORMAT",
+    "TABLE_READ_CLS_BY_FORMAT",
+    "TABLE_WRITE_CLS_BY_FORMAT",
+    "CSVReader",
+    "CSVWriter",
+    "CellValue",
+    "Chunk",
+    "Compression",
+    "DataType",
+    "FailedParsing",
+    "FileReader",
+    "FileWriter",
+    "GzipCompression",
+    "MultiFileReader",
+    "NDJsonReader",
+    "NDJsonWriter",
+    "ParquetReader",
+    "ParquetWriter",
+    "PrimaryCellValue",
+    "SchemaColumn",
+    "Uncompressed",
+    "YAMLReader",
+    "YAMLWriter",
+    "YMLReader",
+    "YMLWriter",
+]

cognite_toolkit/_cdf_tk/utils/fileio/_base.py

@@ -0,0 +1,32 @@
+from abc import ABC
+from dataclasses import dataclass
+from datetime import date, datetime
+from typing import IO, ClassVar, TypeAlias, TypeVar
+
+from cognite_toolkit._cdf_tk.utils.useful_types import DataType, JsonVal
+
+PrimaryCellValue: TypeAlias = datetime | date | JsonVal
+CellValue: TypeAlias = PrimaryCellValue | list[PrimaryCellValue]
+Chunk: TypeAlias = dict[str, CellValue]
+
+
+T_IO = TypeVar("T_IO", bound=IO)
+
+
+class FileIO(ABC):
+    FORMAT: ClassVar[str]
+
+    @property
+    def format(self) -> str:
+        return self.FORMAT
+
+
+@dataclass(frozen=True)
+class SchemaColumn:
+    name: str
+    type: DataType
+    is_array: bool = False
+
+    def __post_init__(self) -> None:
+        if self.type == "json" and self.is_array:
+            raise ValueError("JSON columns cannot be arrays. Use 'is_array=False' for JSON columns.")

cognite_toolkit/_cdf_tk/utils/fileio/_compression.py

@@ -0,0 +1,61 @@
+import gzip
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from io import TextIOWrapper
+from pathlib import Path
+from typing import ClassVar, Literal
+
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.utils import humanize_collection
+from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
+
+
+class Compression(ABC):
+    encoding = "utf-8-sig"
+    newline = "\n"
+    name: ClassVar[str]
+    file_suffix: ClassVar[str]
+
+    def __init__(self, filepath: Path) -> None:
+        self.filepath = filepath
+
+    @abstractmethod
+    def open(self, mode: Literal["r", "w"]) -> TextIOWrapper:
+        """Open the compressed file and return a file-like object."""
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+    @classmethod
+    def from_filepath(cls, filepath: Path) -> "Compression":
+        return COMPRESSION_BY_SUFFIX.get(filepath.suffix, Uncompressed)(filepath=filepath)
+
+    @classmethod
+    def from_name(cls, compression: str) -> "type[Compression]":
+        if compression in COMPRESSION_BY_NAME:
+            return COMPRESSION_BY_NAME[compression]
+        raise ToolkitValueError(
+            f"Unknown compression type: {compression}. Available types: {humanize_collection(COMPRESSION_BY_NAME.keys())}."
+        )
+
+
+class Uncompressed(Compression):
+    name = "none"
+    file_suffix = ""
+
+    def open(self, mode: Literal["r", "w"]) -> TextIOWrapper:
+        """Open the file without compression."""
+        return self.filepath.open(mode=mode, encoding=self.encoding, newline=self.newline)
+
+
+class GzipCompression(Compression):
+    name = "gzip"
+    file_suffix = ".gz"
+
+    def open(self, mode: Literal["r", "w"]) -> TextIOWrapper:
+        """Open the gzip compressed file."""
+        # MyPy (or gzip) fails to recognize that gzip.open returns a TextIOWrapper
+        return gzip.open(self.filepath, mode=f"{mode}t", encoding=self.encoding, newline=self.newline)  # type: ignore[return-value]
+
+
+_all_compression_types: list[type[Compression]] = get_concrete_subclasses(Compression)  # type: ignore[type-abstract]
+COMPRESSION_BY_SUFFIX: Mapping[str, type[Compression]] = {x.file_suffix: x for x in _all_compression_types}
+COMPRESSION_BY_NAME: Mapping[str, type[Compression]] = {x.name: x for x in _all_compression_types}