df-npy 0.1.0__tar.gz

df_npy-0.1.0/.github/workflows/ci-release.yml

@@ -0,0 +1,140 @@
+name: CI and Release
+
+on:
+  pull_request:
+  push:
+    branches:
+      - "**"
+  workflow_dispatch:
+
+jobs:
+  quality:
+    name: Lint, Typecheck, Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Sync dependencies
+        run: uv sync --group dev
+
+      - name: Ruff lint
+        run: uv run ruff check
+
+      - name: Ruff format check
+        run: uv run ruff format --check
+
+      - name: Type check
+        run: uv run ty check
+
+      - name: Test
+        run: uv run python -m pytest -q
+
+  docs:
+    name: Build and Publish Docs
+    needs: quality
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Sync dependencies
+        run: uv sync --group dev
+
+      - name: Build docs
+        run: uv run zensical build
+
+      - name: Deploy to gh-pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages
+          publish_dir: ./site
+          force_orphan: true
+
+  pypi:
+    name: Publish to PyPI (if new version)
+    needs: quality
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Sync dependencies
+        run: uv sync --group dev
+
+      - name: Build package
+        run: uv build
+
+      - name: Check if version already exists on PyPI
+        id: version_check
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          import pathlib
+          import tomllib
+          import urllib.error
+          import urllib.request
+
+          pyproject = tomllib.loads(pathlib.Path("pyproject.toml").read_text())
+          name = pyproject["project"]["name"]
+          version = pyproject["project"]["version"]
+          exists = False
+
+          url = f"https://pypi.org/pypi/{name}/json"
+          try:
+              with urllib.request.urlopen(url, timeout=15) as response:
+                  payload = json.load(response)
+              releases = payload.get("releases", {})
+              exists = version in releases and bool(releases.get(version))
+          except urllib.error.HTTPError as exc:
+              if exc.code != 404:
+                  raise
+
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+              f.write(f"package_name={name}\n")
+              f.write(f"package_version={version}\n")
+              f.write(f"version_exists={str(exists).lower()}\n")
+          PY
+
+      - name: Publish to PyPI with uv (OIDC)
+        if: steps.version_check.outputs.version_exists != 'true'
+        run: uv publish --trusted-publishing always dist/*
+
+      - name: Report skipped publish
+        if: steps.version_check.outputs.version_exists == 'true'
+        run: |
+          echo "PyPI publish skipped."
+          echo "version_exists=${{ steps.version_check.outputs.version_exists }}"

df_npy-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+*.pyc
+__pycache__/
+site/
+build/
+dist/
+*.egg-info/

df_npy-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.20
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: ty-check
+        name: ty check
+        entry: uv run ty check
+        language: system
+        pass_filenames: false
+        types: [python]

df_npy-0.1.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: df-npy
+Version: 0.1.0
+Requires-Python: >=3.14
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: numpy>=2.5.0
+Requires-Dist: pandas>=3.0.3

df_npy-0.1.0/README.md

@@ -0,0 +1,39 @@
+# df-npy
+
+Serialize and deserialize pandas DataFrames to `.npy` plus JSON metadata.
+
+## Security
+
+### Pickle policy
+
+`df-npy` has a strict no-pickle policy.
+
+- Serialization always writes NumPy arrays with `allow_pickle=False`.
+- Deserialization always reads with `allow_pickle=False`.
+- DataFrames that would require pickle-backed object serialization are rejected.
+
+This behavior is intentional for security-sensitive environments.
+
+### What this means in practice
+
+Supported data should be representable without Python object pickling.
+
+Examples of unsupported frames:
+
+- Object/string frames containing non-string Python objects.
+- Heterogeneous object columns that rely on pickle to round-trip values.
+
+When unsupported data is provided, `df-npy` raises `ValueError` rather than falling back to pickle.
+
+## API
+
+Public API:
+
+- `NpySerializer.to_npy(df, file_path)`
+- `NpySerializer.from_npy(file_path, identifiers=None)`
+
+Import:
+
+```python
+from df_npy import NpySerializer
+```

df_npy-0.1.0/df_npy/__init__.py

@@ -0,0 +1,5 @@
+"""Public package API for df_npy."""
+
+from ._serializer import NpySerializer
+
+__all__ = ["NpySerializer"]

df_npy-0.1.0/df_npy/_arrays.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from ._constants import (
+    NUMPY_DTYPE_FLOAT64,
+    NUMPY_DTYPE_UNICODE,
+    STRING_MISSING_VALUE_SENTINEL,
+)
+from ._dtypes import DtypePlan, is_string_dtype
+
+
+def prepare_writable_array(df: pd.DataFrame, plan: DtypePlan) -> np.ndarray:
+    if is_string_dtype(plan.representative_dtype):
+        array = (
+            df.astype(object)
+            .where(df.notna(), other=STRING_MISSING_VALUE_SENTINEL)
+            .to_numpy(dtype=NUMPY_DTYPE_UNICODE, copy=False)
+        )
+        return np.asfortranarray(array)
+
+    if plan.mixed_numeric:
+        return np.asfortranarray(df.to_numpy(dtype=NUMPY_DTYPE_FLOAT64, copy=False))
+
+    return np.asfortranarray(df.to_numpy(dtype=plan.representative_dtype, copy=False))

df_npy-0.1.0/df_npy/_axis.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import re
+from contextlib import suppress
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ._constants import DEFAULT_TIME_UNIT, AxisMetaKey, AxisType
+
+
+def _extract_time_unit(dtype_name: str | None, default: str = DEFAULT_TIME_UNIT) -> str:
+    if not dtype_name:
+        return default
+    match = re.search(r"\[(\w+)", dtype_name)
+    if match:
+        return match.group(1)
+    return default
+
+
+def serialise_axis(index: pd.Index | pd.MultiIndex) -> dict[str, Any]:
+    if isinstance(index, pd.MultiIndex):
+        return {
+            AxisMetaKey.TYPE.value: AxisType.MULTIINDEX.value,
+            AxisMetaKey.NAMES.value: list(index.names),
+            AxisMetaKey.NLEVELS.value: index.nlevels,
+            AxisMetaKey.LEVELS.value: [serialise_axis(level) for level in index.levels],
+            AxisMetaKey.CODES.value: [codes.tolist() for codes in index.codes],
+            AxisMetaKey.SORTORDER.value: index.sortorder,
+        }
+
+    payload: dict[str, Any] = {
+        AxisMetaKey.TYPE.value: type(index).__name__,
+        AxisMetaKey.NAME.value: index.name,
+    }
+
+    if isinstance(index, pd.RangeIndex):
+        payload.update(
+            {
+                AxisMetaKey.RANGE.value: True,
+                AxisMetaKey.START.value: int(index.start),
+                AxisMetaKey.STOP.value: int(index.stop),
+                AxisMetaKey.STEP.value: int(index.step),
+            },
+        )
+        return payload
+
+    if isinstance(index, pd.DatetimeIndex):
+        payload.update(
+            {
+                AxisMetaKey.DATETIME.value: True,
+                AxisMetaKey.DTYPE.value: index.dtype.name,
+                AxisMetaKey.TZ.value: str(index.tz) if index.tz is not None else None,
+                AxisMetaKey.FREQ.value: index.freqstr,
+                AxisMetaKey.VALUES_I8.value: index.asi8.tolist(),
+            },
+        )
+        return payload
+
+    if isinstance(index, pd.TimedeltaIndex):
+        payload.update(
+            {
+                AxisMetaKey.TIMEDELTA.value: True,
+                AxisMetaKey.DTYPE.value: index.dtype.name,
+                AxisMetaKey.FREQ.value: index.freqstr,
+                AxisMetaKey.VALUES_I8.value: index.asi8.tolist(),
+            },
+        )
+        return payload
+
+    payload.update(
+        {
+            AxisMetaKey.VALUES.value: index.tolist(),
+            AxisMetaKey.DTYPE.value: str(getattr(index, "dtype", "object")),
+        },
+    )
+    return payload
+
+
+def deserialise_axis(metadata: dict[str, Any]) -> pd.Index | pd.MultiIndex:
+    axis_type = metadata.get(AxisMetaKey.TYPE.value)
+
+    if axis_type == AxisType.MULTIINDEX.value:
+        levels = [
+            deserialise_axis(level) for level in metadata[AxisMetaKey.LEVELS.value]
+        ]
+        return pd.MultiIndex(
+            levels=levels,
+            codes=metadata[AxisMetaKey.CODES.value],
+            names=metadata.get(AxisMetaKey.NAMES.value),
+            sortorder=metadata.get(AxisMetaKey.SORTORDER.value),
+        )
+
+    if metadata.get(AxisMetaKey.RANGE.value):
+        return pd.RangeIndex(
+            start=metadata[AxisMetaKey.START.value],
+            stop=metadata[AxisMetaKey.STOP.value],
+            step=metadata[AxisMetaKey.STEP.value],
+            name=metadata.get(AxisMetaKey.NAME.value),
+        )
+
+    if metadata.get(AxisMetaKey.DATETIME.value):
+        unit = _extract_time_unit(metadata.get(AxisMetaKey.DTYPE.value))
+        values = metadata.get(AxisMetaKey.VALUES_I8.value, [])
+        tz = metadata.get(AxisMetaKey.TZ.value)
+
+        if tz:
+            idx = pd.to_datetime(values, unit=unit, utc=True)
+            idx = pd.DatetimeIndex(
+                idx,
+                name=metadata.get(AxisMetaKey.NAME.value),
+            ).tz_convert(tz)
+        else:
+            idx = pd.DatetimeIndex(
+                pd.to_datetime(values, unit=unit),
+                name=metadata.get(AxisMetaKey.NAME.value),
+            )
+
+        if freq := metadata.get(AxisMetaKey.FREQ.value):
+            with suppress(ValueError):
+                idx = pd.DatetimeIndex(idx, name=idx.name, freq=freq)
+        return idx
+
+    if metadata.get(AxisMetaKey.TIMEDELTA.value):
+        unit = _extract_time_unit(metadata.get(AxisMetaKey.DTYPE.value))
+        values_i8 = metadata.get(AxisMetaKey.VALUES_I8.value, [])
+        td_arr = np.array(values_i8, dtype=f"timedelta64[{unit}]")
+        idx = pd.TimedeltaIndex(
+            td_arr,
+            name=metadata.get(AxisMetaKey.NAME.value),
+        )
+        if freq := metadata.get(AxisMetaKey.FREQ.value):
+            with suppress(ValueError):
+                idx = pd.TimedeltaIndex(idx, name=idx.name, freq=freq)
+        return idx
+
+    return pd.Index(
+        metadata.get(AxisMetaKey.VALUES.value, []),
+        name=metadata.get(AxisMetaKey.NAME.value),
+    )

df_npy-0.1.0/df_npy/_constants.py

@@ -0,0 +1,51 @@
+from enum import StrEnum
+
+
+class MetadataKey(StrEnum):
+    COLUMNS = "columns"
+    INDEX = "index"
+    DTYPE = "dtype"
+    STORAGE_DTYPE = "storage_dtype"
+    COLUMN_DTYPES = "column_dtypes"
+    SHAPE = "shape"
+    ORDER = "order"
+
+
+class AxisMetaKey(StrEnum):
+    TYPE = "type"
+    NAME = "name"
+    RANGE = "range"
+    START = "start"
+    STOP = "stop"
+    STEP = "step"
+    DATETIME = "datetime"
+    TIMEDELTA = "timedelta"
+    DTYPE = "dtype"
+    TZ = "tz"
+    FREQ = "freq"
+    VALUES_I8 = "values_i8"
+    VALUES = "values"
+    LEVELS = "levels"
+    CODES = "codes"
+    NAMES = "names"
+    SORTORDER = "sortorder"
+    NLEVELS = "nlevels"
+
+
+class AxisType(StrEnum):
+    MULTIINDEX = "multiindex"
+
+
+STRING_MISSING_VALUE_SENTINEL = "<MISSING_STRING_VALUE_SENTINEL_df_npy>"
+NUMPY_DTYPE_FLOAT64 = "float64"
+NUMPY_DTYPE_UNICODE = "U"
+PANDAS_NULLABLE_INT_DTYPE = "Int64"
+PANDAS_BOOL_DTYPE = "bool"
+PANDAS_NULLABLE_BOOL_DTYPE = "boolean"
+DEFAULT_TIME_UNIT = "ns"
+NPY_SUFFIX = ".npy"
+JSON_SUFFIX = ".json"
+ARRAY_ORDER_FORTRAN = "F"
+
+
+STRING_DTYPE_NAMES = {"object", "string", "str"}

df_npy-0.1.0/df_npy/_dtypes.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import pandas as pd
+from numpy.typing import DTypeLike
+
+from ._constants import (
+    NUMPY_DTYPE_FLOAT64,
+    PANDAS_BOOL_DTYPE,
+    PANDAS_NULLABLE_BOOL_DTYPE,
+    STRING_DTYPE_NAMES,
+)
+
+
+@dataclass(frozen=True)
+class DtypePlan:
+    representative_dtype: DTypeLike
+    mixed_numeric: bool
+    column_dtypes: list[str]
+
+
+def is_string_dtype(dtype: object) -> bool:
+    if dtype is None:
+        return False
+    try:
+        return pd.api.types.is_string_dtype(dtype) or pd.api.types.is_object_dtype(
+            dtype,
+        )
+    except TypeError, ValueError:
+        return str(dtype) in STRING_DTYPE_NAMES
+
+
+def is_numeric_dtype(dtype: object) -> bool:
+    try:
+        return pd.api.types.is_numeric_dtype(dtype)
+    except TypeError, ValueError:
+        return False
+
+
+def is_integer_dtype(dtype: object) -> bool:
+    try:
+        return pd.api.types.is_integer_dtype(dtype)
+    except TypeError, ValueError:
+        return False
+
+
+def is_float_dtype(dtype: object) -> bool:
+    try:
+        return pd.api.types.is_float_dtype(dtype)
+    except TypeError, ValueError:
+        return False
+
+
+def extract_dtype_plan(df: pd.DataFrame) -> DtypePlan:
+    column_dtypes = [str(dtype) for dtype in df.dtypes]
+    dtypes = df.dtypes
+    n_distinct_dtypes = dtypes.nunique()
+
+    if n_distinct_dtypes == 0:
+        return DtypePlan(
+            representative_dtype=NUMPY_DTYPE_FLOAT64,
+            mixed_numeric=False,
+            column_dtypes=column_dtypes,
+        )
+
+    if any(is_string_dtype(dt) for dt in dtypes):
+        if n_distinct_dtypes > 1:
+            msg = (
+                f"DataFrame has {n_distinct_dtypes} distinct dtypes; "
+                "only single-dtype frames are supported "
+                "(string/object cannot be mixed)."
+            )
+            raise ValueError(
+                msg,
+            )
+        non_null = df.stack().dropna()
+        has_non_string = not non_null.map(lambda value: isinstance(value, str)).all()
+        if has_non_string:
+            msg = (
+                "Pickle-backed object serialization is disabled; "
+                "object/string frames must contain only string values "
+                "and missing values."
+            )
+            raise ValueError(
+                msg,
+            )
+        return DtypePlan(
+            representative_dtype=dtypes.iloc[0],
+            mixed_numeric=False,
+            column_dtypes=column_dtypes,
+        )
+
+    if all(is_numeric_dtype(dt) for dt in dtypes):
+        has_int = any(is_integer_dtype(dt) for dt in dtypes)
+        has_float = any(is_float_dtype(dt) for dt in dtypes)
+        if has_int and has_float:
+            return DtypePlan(
+                representative_dtype=NUMPY_DTYPE_FLOAT64,
+                mixed_numeric=True,
+                column_dtypes=column_dtypes,
+            )
+        if n_distinct_dtypes > 1:
+            msg = (
+                f"DataFrame has {n_distinct_dtypes} distinct numeric dtypes; "
+                "only int+float mixing is supported."
+            )
+            raise ValueError(
+                msg,
+            )
+        return DtypePlan(
+            representative_dtype=dtypes.iloc[0],
+            mixed_numeric=False,
+            column_dtypes=column_dtypes,
+        )
+
+    if n_distinct_dtypes > 1:
+        msg = (
+            f"DataFrame has {n_distinct_dtypes} distinct dtypes; "
+            "only single-dtype frames are supported."
+        )
+        raise ValueError(
+            msg,
+        )
+
+    return DtypePlan(
+        representative_dtype=dtypes.iloc[0],
+        mixed_numeric=False,
+        column_dtypes=column_dtypes,
+    )
+
+
+def _nullable_integer_dtype(dtype_name: str) -> str:
+    match = re.fullmatch(r"(u?)int(8|16|32|64)", dtype_name)
+    if not match:
+        return "Int64"
+    unsigned, bits = match.groups()
+    prefix = "UInt" if unsigned else "Int"
+    return f"{prefix}{bits}"
+
+
+def restore_column_dtypes(df: pd.DataFrame, column_dtypes: list[str]) -> pd.DataFrame:
+    if len(column_dtypes) != len(df.columns):
+        raise ValueError(
+            "column_dtypes length does not match DataFrame columns length.",
+        )
+
+    for position, column in enumerate(df.columns):
+        dtype_name = column_dtypes[position]
+        series = df[column]
+        if dtype_name.startswith(("int", "uint")):
+            if series.isna().any():
+                nullable_dtype = _nullable_integer_dtype(dtype_name)
+                df[column] = pd.to_numeric(series, errors="coerce").astype(
+                    nullable_dtype,
+                )
+            else:
+                df[column] = pd.to_numeric(series, errors="raise").astype(dtype_name)
+        elif dtype_name.startswith("float"):
+            df[column] = pd.to_numeric(series, errors="coerce").astype(dtype_name)
+        elif dtype_name == PANDAS_BOOL_DTYPE:
+            df[column] = series.astype(PANDAS_BOOL_DTYPE)
+        elif dtype_name == PANDAS_NULLABLE_BOOL_DTYPE:
+            df[column] = series.astype(PANDAS_NULLABLE_BOOL_DTYPE)
+        else:
+            df[column] = series.astype(dtype_name)
+
+    return df

df_npy-0.1.0/df_npy/_json.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import numpy as np
+
+
+def safe_str(obj: object) -> str:
+    try:
+        return str(obj)
+    except Exception:
+        return repr(obj)
+
+
+def json_default(obj: object) -> bool | float | int | str:
+    if isinstance(obj, np.integer):
+        return int(obj)
+    if isinstance(obj, np.floating):
+        return float(obj)
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    if isinstance(obj, (np.datetime64, np.timedelta64)):
+        return str(obj)
+    return safe_str(obj)

df_npy-0.1.0/df_npy/_paths.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ._constants import JSON_SUFFIX, NPY_SUFFIX
+
+
+def ensure_npy_path(file_path: Path | str, *, for_write: bool) -> Path:
+    path = Path(file_path)
+    if path.suffix != NPY_SUFFIX:
+        path = path.with_suffix(NPY_SUFFIX)
+    if for_write:
+        path.parent.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def metadata_file_from_npy_file(npy_file: Path) -> Path:
+    return npy_file.with_suffix(JSON_SUFFIX)