df-npy 0.1.0__py3-none-any.whl

df_npy/__init__.py

@@ -0,0 +1,5 @@
+"""Public package API for df_npy."""
+
+from ._serializer import NpySerializer
+
+__all__ = ["NpySerializer"]

df_npy/_arrays.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from ._constants import (
+    NUMPY_DTYPE_FLOAT64,
+    NUMPY_DTYPE_UNICODE,
+    STRING_MISSING_VALUE_SENTINEL,
+)
+from ._dtypes import DtypePlan, is_string_dtype
+
+
+def prepare_writable_array(df: pd.DataFrame, plan: DtypePlan) -> np.ndarray:
+    if is_string_dtype(plan.representative_dtype):
+        array = (
+            df.astype(object)
+            .where(df.notna(), other=STRING_MISSING_VALUE_SENTINEL)
+            .to_numpy(dtype=NUMPY_DTYPE_UNICODE, copy=False)
+        )
+        return np.asfortranarray(array)
+
+    if plan.mixed_numeric:
+        return np.asfortranarray(df.to_numpy(dtype=NUMPY_DTYPE_FLOAT64, copy=False))
+
+    return np.asfortranarray(df.to_numpy(dtype=plan.representative_dtype, copy=False))

df_npy/_axis.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import re
+from contextlib import suppress
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ._constants import DEFAULT_TIME_UNIT, AxisMetaKey, AxisType
+
+
+def _extract_time_unit(dtype_name: str | None, default: str = DEFAULT_TIME_UNIT) -> str:
+    if not dtype_name:
+        return default
+    match = re.search(r"\[(\w+)", dtype_name)
+    if match:
+        return match.group(1)
+    return default
+
+
+def serialise_axis(index: pd.Index | pd.MultiIndex) -> dict[str, Any]:
+    if isinstance(index, pd.MultiIndex):
+        return {
+            AxisMetaKey.TYPE.value: AxisType.MULTIINDEX.value,
+            AxisMetaKey.NAMES.value: list(index.names),
+            AxisMetaKey.NLEVELS.value: index.nlevels,
+            AxisMetaKey.LEVELS.value: [serialise_axis(level) for level in index.levels],
+            AxisMetaKey.CODES.value: [codes.tolist() for codes in index.codes],
+            AxisMetaKey.SORTORDER.value: index.sortorder,
+        }
+
+    payload: dict[str, Any] = {
+        AxisMetaKey.TYPE.value: type(index).__name__,
+        AxisMetaKey.NAME.value: index.name,
+    }
+
+    if isinstance(index, pd.RangeIndex):
+        payload.update(
+            {
+                AxisMetaKey.RANGE.value: True,
+                AxisMetaKey.START.value: int(index.start),
+                AxisMetaKey.STOP.value: int(index.stop),
+                AxisMetaKey.STEP.value: int(index.step),
+            },
+        )
+        return payload
+
+    if isinstance(index, pd.DatetimeIndex):
+        payload.update(
+            {
+                AxisMetaKey.DATETIME.value: True,
+                AxisMetaKey.DTYPE.value: index.dtype.name,
+                AxisMetaKey.TZ.value: str(index.tz) if index.tz is not None else None,
+                AxisMetaKey.FREQ.value: index.freqstr,
+                AxisMetaKey.VALUES_I8.value: index.asi8.tolist(),
+            },
+        )
+        return payload
+
+    if isinstance(index, pd.TimedeltaIndex):
+        payload.update(
+            {
+                AxisMetaKey.TIMEDELTA.value: True,
+                AxisMetaKey.DTYPE.value: index.dtype.name,
+                AxisMetaKey.FREQ.value: index.freqstr,
+                AxisMetaKey.VALUES_I8.value: index.asi8.tolist(),
+            },
+        )
+        return payload
+
+    payload.update(
+        {
+            AxisMetaKey.VALUES.value: index.tolist(),
+            AxisMetaKey.DTYPE.value: str(getattr(index, "dtype", "object")),
+        },
+    )
+    return payload
+
+
+def deserialise_axis(metadata: dict[str, Any]) -> pd.Index | pd.MultiIndex:
+    axis_type = metadata.get(AxisMetaKey.TYPE.value)
+
+    if axis_type == AxisType.MULTIINDEX.value:
+        levels = [
+            deserialise_axis(level) for level in metadata[AxisMetaKey.LEVELS.value]
+        ]
+        return pd.MultiIndex(
+            levels=levels,
+            codes=metadata[AxisMetaKey.CODES.value],
+            names=metadata.get(AxisMetaKey.NAMES.value),
+            sortorder=metadata.get(AxisMetaKey.SORTORDER.value),
+        )
+
+    if metadata.get(AxisMetaKey.RANGE.value):
+        return pd.RangeIndex(
+            start=metadata[AxisMetaKey.START.value],
+            stop=metadata[AxisMetaKey.STOP.value],
+            step=metadata[AxisMetaKey.STEP.value],
+            name=metadata.get(AxisMetaKey.NAME.value),
+        )
+
+    if metadata.get(AxisMetaKey.DATETIME.value):
+        unit = _extract_time_unit(metadata.get(AxisMetaKey.DTYPE.value))
+        values = metadata.get(AxisMetaKey.VALUES_I8.value, [])
+        tz = metadata.get(AxisMetaKey.TZ.value)
+
+        if tz:
+            idx = pd.to_datetime(values, unit=unit, utc=True)
+            idx = pd.DatetimeIndex(
+                idx,
+                name=metadata.get(AxisMetaKey.NAME.value),
+            ).tz_convert(tz)
+        else:
+            idx = pd.DatetimeIndex(
+                pd.to_datetime(values, unit=unit),
+                name=metadata.get(AxisMetaKey.NAME.value),
+            )
+
+        if freq := metadata.get(AxisMetaKey.FREQ.value):
+            with suppress(ValueError):
+                idx = pd.DatetimeIndex(idx, name=idx.name, freq=freq)
+        return idx
+
+    if metadata.get(AxisMetaKey.TIMEDELTA.value):
+        unit = _extract_time_unit(metadata.get(AxisMetaKey.DTYPE.value))
+        values_i8 = metadata.get(AxisMetaKey.VALUES_I8.value, [])
+        td_arr = np.array(values_i8, dtype=f"timedelta64[{unit}]")
+        idx = pd.TimedeltaIndex(
+            td_arr,
+            name=metadata.get(AxisMetaKey.NAME.value),
+        )
+        if freq := metadata.get(AxisMetaKey.FREQ.value):
+            with suppress(ValueError):
+                idx = pd.TimedeltaIndex(idx, name=idx.name, freq=freq)
+        return idx
+
+    return pd.Index(
+        metadata.get(AxisMetaKey.VALUES.value, []),
+        name=metadata.get(AxisMetaKey.NAME.value),
+    )

df_npy/_constants.py

@@ -0,0 +1,51 @@
+from enum import StrEnum
+
+
+class MetadataKey(StrEnum):
+    COLUMNS = "columns"
+    INDEX = "index"
+    DTYPE = "dtype"
+    STORAGE_DTYPE = "storage_dtype"
+    COLUMN_DTYPES = "column_dtypes"
+    SHAPE = "shape"
+    ORDER = "order"
+
+
+class AxisMetaKey(StrEnum):
+    TYPE = "type"
+    NAME = "name"
+    RANGE = "range"
+    START = "start"
+    STOP = "stop"
+    STEP = "step"
+    DATETIME = "datetime"
+    TIMEDELTA = "timedelta"
+    DTYPE = "dtype"
+    TZ = "tz"
+    FREQ = "freq"
+    VALUES_I8 = "values_i8"
+    VALUES = "values"
+    LEVELS = "levels"
+    CODES = "codes"
+    NAMES = "names"
+    SORTORDER = "sortorder"
+    NLEVELS = "nlevels"
+
+
+class AxisType(StrEnum):
+    MULTIINDEX = "multiindex"
+
+
+STRING_MISSING_VALUE_SENTINEL = "<MISSING_STRING_VALUE_SENTINEL_df_npy>"
+NUMPY_DTYPE_FLOAT64 = "float64"
+NUMPY_DTYPE_UNICODE = "U"
+PANDAS_NULLABLE_INT_DTYPE = "Int64"
+PANDAS_BOOL_DTYPE = "bool"
+PANDAS_NULLABLE_BOOL_DTYPE = "boolean"
+DEFAULT_TIME_UNIT = "ns"
+NPY_SUFFIX = ".npy"
+JSON_SUFFIX = ".json"
+ARRAY_ORDER_FORTRAN = "F"
+
+
+STRING_DTYPE_NAMES = {"object", "string", "str"}

df_npy/_dtypes.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import pandas as pd
+from numpy.typing import DTypeLike
+
+from ._constants import (
+    NUMPY_DTYPE_FLOAT64,
+    PANDAS_BOOL_DTYPE,
+    PANDAS_NULLABLE_BOOL_DTYPE,
+    STRING_DTYPE_NAMES,
+)
+
+
+@dataclass(frozen=True)
+class DtypePlan:
+    representative_dtype: DTypeLike
+    mixed_numeric: bool
+    column_dtypes: list[str]
+
+
+def is_string_dtype(dtype: object) -> bool:
+    if dtype is None:
+        return False
+    try:
+        return pd.api.types.is_string_dtype(dtype) or pd.api.types.is_object_dtype(
+            dtype,
+        )
+    except TypeError, ValueError:
+        return str(dtype) in STRING_DTYPE_NAMES
+
+
+def is_numeric_dtype(dtype: object) -> bool:
+    try:
+        return pd.api.types.is_numeric_dtype(dtype)
+    except TypeError, ValueError:
+        return False
+
+
+def is_integer_dtype(dtype: object) -> bool:
+    try:
+        return pd.api.types.is_integer_dtype(dtype)
+    except TypeError, ValueError:
+        return False
+
+
+def is_float_dtype(dtype: object) -> bool:
+    try:
+        return pd.api.types.is_float_dtype(dtype)
+    except TypeError, ValueError:
+        return False
+
+
+def extract_dtype_plan(df: pd.DataFrame) -> DtypePlan:
+    column_dtypes = [str(dtype) for dtype in df.dtypes]
+    dtypes = df.dtypes
+    n_distinct_dtypes = dtypes.nunique()
+
+    if n_distinct_dtypes == 0:
+        return DtypePlan(
+            representative_dtype=NUMPY_DTYPE_FLOAT64,
+            mixed_numeric=False,
+            column_dtypes=column_dtypes,
+        )
+
+    if any(is_string_dtype(dt) for dt in dtypes):
+        if n_distinct_dtypes > 1:
+            msg = (
+                f"DataFrame has {n_distinct_dtypes} distinct dtypes; "
+                "only single-dtype frames are supported "
+                "(string/object cannot be mixed)."
+            )
+            raise ValueError(
+                msg,
+            )
+        non_null = df.stack().dropna()
+        has_non_string = not non_null.map(lambda value: isinstance(value, str)).all()
+        if has_non_string:
+            msg = (
+                "Pickle-backed object serialization is disabled; "
+                "object/string frames must contain only string values "
+                "and missing values."
+            )
+            raise ValueError(
+                msg,
+            )
+        return DtypePlan(
+            representative_dtype=dtypes.iloc[0],
+            mixed_numeric=False,
+            column_dtypes=column_dtypes,
+        )
+
+    if all(is_numeric_dtype(dt) for dt in dtypes):
+        has_int = any(is_integer_dtype(dt) for dt in dtypes)
+        has_float = any(is_float_dtype(dt) for dt in dtypes)
+        if has_int and has_float:
+            return DtypePlan(
+                representative_dtype=NUMPY_DTYPE_FLOAT64,
+                mixed_numeric=True,
+                column_dtypes=column_dtypes,
+            )
+        if n_distinct_dtypes > 1:
+            msg = (
+                f"DataFrame has {n_distinct_dtypes} distinct numeric dtypes; "
+                "only int+float mixing is supported."
+            )
+            raise ValueError(
+                msg,
+            )
+        return DtypePlan(
+            representative_dtype=dtypes.iloc[0],
+            mixed_numeric=False,
+            column_dtypes=column_dtypes,
+        )
+
+    if n_distinct_dtypes > 1:
+        msg = (
+            f"DataFrame has {n_distinct_dtypes} distinct dtypes; "
+            "only single-dtype frames are supported."
+        )
+        raise ValueError(
+            msg,
+        )
+
+    return DtypePlan(
+        representative_dtype=dtypes.iloc[0],
+        mixed_numeric=False,
+        column_dtypes=column_dtypes,
+    )
+
+
+def _nullable_integer_dtype(dtype_name: str) -> str:
+    match = re.fullmatch(r"(u?)int(8|16|32|64)", dtype_name)
+    if not match:
+        return "Int64"
+    unsigned, bits = match.groups()
+    prefix = "UInt" if unsigned else "Int"
+    return f"{prefix}{bits}"
+
+
+def restore_column_dtypes(df: pd.DataFrame, column_dtypes: list[str]) -> pd.DataFrame:
+    if len(column_dtypes) != len(df.columns):
+        raise ValueError(
+            "column_dtypes length does not match DataFrame columns length.",
+        )
+
+    for position, column in enumerate(df.columns):
+        dtype_name = column_dtypes[position]
+        series = df[column]
+        if dtype_name.startswith(("int", "uint")):
+            if series.isna().any():
+                nullable_dtype = _nullable_integer_dtype(dtype_name)
+                df[column] = pd.to_numeric(series, errors="coerce").astype(
+                    nullable_dtype,
+                )
+            else:
+                df[column] = pd.to_numeric(series, errors="raise").astype(dtype_name)
+        elif dtype_name.startswith("float"):
+            df[column] = pd.to_numeric(series, errors="coerce").astype(dtype_name)
+        elif dtype_name == PANDAS_BOOL_DTYPE:
+            df[column] = series.astype(PANDAS_BOOL_DTYPE)
+        elif dtype_name == PANDAS_NULLABLE_BOOL_DTYPE:
+            df[column] = series.astype(PANDAS_NULLABLE_BOOL_DTYPE)
+        else:
+            df[column] = series.astype(dtype_name)
+
+    return df

df_npy/_json.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import numpy as np
+
+
+def safe_str(obj: object) -> str:
+    try:
+        return str(obj)
+    except Exception:
+        return repr(obj)
+
+
+def json_default(obj: object) -> bool | float | int | str:
+    if isinstance(obj, np.integer):
+        return int(obj)
+    if isinstance(obj, np.floating):
+        return float(obj)
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    if isinstance(obj, (np.datetime64, np.timedelta64)):
+        return str(obj)
+    return safe_str(obj)

df_npy/_paths.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ._constants import JSON_SUFFIX, NPY_SUFFIX
+
+
+def ensure_npy_path(file_path: Path | str, *, for_write: bool) -> Path:
+    path = Path(file_path)
+    if path.suffix != NPY_SUFFIX:
+        path = path.with_suffix(NPY_SUFFIX)
+    if for_write:
+        path.parent.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def metadata_file_from_npy_file(npy_file: Path) -> Path:
+    return npy_file.with_suffix(JSON_SUFFIX)

df_npy/_serializer.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from loguru import logger
+
+from ._arrays import prepare_writable_array
+from ._axis import deserialise_axis, serialise_axis
+from ._constants import ARRAY_ORDER_FORTRAN, STRING_MISSING_VALUE_SENTINEL, MetadataKey
+from ._dtypes import extract_dtype_plan, is_string_dtype, restore_column_dtypes
+from ._json import json_default
+from ._paths import ensure_npy_path, metadata_file_from_npy_file
+
+
+class NpySerializer:
+    @classmethod
+    def to_npy(cls, df: pd.DataFrame, file_path: Path | str) -> None:
+        path = ensure_npy_path(file_path, for_write=True)
+        if not df.columns.is_unique:
+            raise ValueError("Columns must be unique for identifier-based subsetting.")
+
+        logger.info(f"Serializing DataFrame to {path}")
+        logger.debug("Extracting dtype from DataFrame")
+        plan = extract_dtype_plan(df)
+
+        logger.debug("Converting DataFrame to Fortran-ordered NumPy array")
+        np_array = prepare_writable_array(df, plan)
+
+        logger.debug("Creating metadata for DataFrame serialization")
+        metadata = {
+            MetadataKey.COLUMNS.value: serialise_axis(df.columns),
+            MetadataKey.INDEX.value: serialise_axis(df.index),
+            MetadataKey.DTYPE.value: str(plan.representative_dtype),
+            MetadataKey.STORAGE_DTYPE.value: str(np_array.dtype),
+            MetadataKey.COLUMN_DTYPES.value: plan.column_dtypes,
+            MetadataKey.SHAPE.value: list(np_array.shape),
+            MetadataKey.ORDER.value: ARRAY_ORDER_FORTRAN,
+        }
+
+        logger.debug("Saving array")
+        np.save(path, np_array, allow_pickle=False)
+
+        logger.debug("Saving metadata")
+        metadata_path = metadata_file_from_npy_file(path)
+        metadata_path.write_text(
+            json.dumps(metadata, indent=4, default=json_default),
+            encoding="utf-8",
+        )
+        logger.success(f"DataFrame serialized to {path}.")
+
+    @classmethod
+    def from_npy(
+        cls,
+        file_path: Path | str,
+        *,
+        identifiers: list[str] | None = None,
+    ) -> pd.DataFrame:
+        path = ensure_npy_path(file_path, for_write=False)
+        logger.info(f"Deserializing DataFrame from {path}")
+        if not path.exists():
+            raise FileNotFoundError(path)
+
+        logger.debug("Loading metadata from JSON file")
+        metadata_path = metadata_file_from_npy_file(path)
+        metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+
+        if MetadataKey.COLUMN_DTYPES.value not in metadata:
+            raise ValueError("Metadata is missing required key: column_dtypes")
+        column_dtypes = metadata[MetadataKey.COLUMN_DTYPES.value]
+
+        logger.debug("Loading NumPy array from file")
+        np_array = np.load(path, allow_pickle=False, mmap_mode="r")
+        if not np_array.flags["F_CONTIGUOUS"]:
+            raise ValueError("Stored array is not Fortran contiguous as expected.")
+
+        logger.debug("Deserializing index")
+        index = deserialise_axis(metadata[MetadataKey.INDEX.value])
+
+        logger.debug("Deserializing columns")
+        columns = deserialise_axis(metadata[MetadataKey.COLUMNS.value])
+
+        if identifiers is not None:
+            logger.debug("Subsetting columns based on provided identifiers")
+            positions = {column: position for position, column in enumerate(columns)}
+            try:
+                column_indices = [positions[column] for column in identifiers]
+            except KeyError as exc:
+                raise KeyError(f"Identifier not found in columns: {exc}") from None
+            np_array = np_array[:, np.asarray(column_indices, dtype=np.int64)]
+            columns = pd.Index(identifiers, name=columns.name)
+            column_dtypes = [column_dtypes[index] for index in column_indices]
+        else:
+            _ = np_array[...]
+
+        logger.debug("Creating DataFrame from NumPy array, columns, and index")
+        df = pd.DataFrame(np_array, columns=columns, index=index, copy=False)
+
+        if is_string_dtype(metadata.get(MetadataKey.DTYPE.value)):
+            logger.debug("Replacing sentinel values with NaN for string/object dtype")
+            df.replace(STRING_MISSING_VALUE_SENTINEL, np.nan, inplace=True)
+
+        logger.debug("Restoring per-column dtypes from metadata")
+        df = restore_column_dtypes(df, column_dtypes)
+
+        logger.success(f"DataFrame deserialized from {path}.")
+        logger.debug(f"Deserialized DataFrame shape: {df.shape}")
+        return df

df_npy-0.1.0.dist-info/METADATA

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: df-npy
+Version: 0.1.0
+Requires-Python: >=3.14
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: numpy>=2.5.0
+Requires-Dist: pandas>=3.0.3

df_npy-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+df_npy/__init__.py,sha256=Ox1NtpAz9uHwGMOw6cz0qXed8RBemmxMTsBd9y0F3g4,106
+df_npy/_arrays.py,sha256=8dmfaI7iRQh9KmONoUTSwsLIrUh8i4obZwpvWHzTgNM,852
+df_npy/_axis.py,sha256=tJVwl4eyRMuSuNSecPu27VxsXsASSBBFMZue8Vz1mf0,4828
+df_npy/_constants.py,sha256=vO86zJvEavu2p4owU1j0jpP3hNDVhVHTY0-htxuOjjU,1079
+df_npy/_dtypes.py,sha256=pDlP7bsERr035gLzb3QWpWNA_IzQBBLqqjMGlvgQEug,5152
+df_npy/_json.py,sha256=knr5EsUnj88L6d9dVEUDu5YNNknWUu_0JH4WrbSQ1kI,520
+df_npy/_paths.py,sha256=XiGRhZxQPKPiYVQmhAAVqvpf65kl_vdPG-PfS8aevZs,482
+df_npy/_serializer.py,sha256=Yn0v9eYaLYejlvgklLEAkNCYLDsLORF7MUcWyZpqDkU,4583
+df_npy-0.1.0.dist-info/METADATA,sha256=A82OGlNX-seTKYIw7eJxxO-ydxcqFuTAS9bAra88xXM,160
+df_npy-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+df_npy-0.1.0.dist-info/RECORD,,

df_npy-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any