colstore 0.1.0__tar.gz

colstore-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,93 @@
+name: CI
+
+# Run on every push to main and on every PR targeting main. Other
+# branches don't trigger CI by default — open a PR to get feedback.
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+# Cancel in-progress runs for the same ref when a new push arrives.
+# Saves CI minutes and gets you the "latest commit only" result.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+# Default to read-only permissions; release.yml elevates as needed.
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint and type-check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dev dependencies
+        # Editable install also compiles the C++ extension via
+        # scikit-build-core; mypy needs the module importable to typecheck
+        # against it.
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: ruff (lint)
+        run: ruff check src tests
+
+      - name: black (format check)
+        run: black --check src tests
+
+      - name: mypy (strict type check)
+        run: mypy src
+
+  test:
+    name: Test (Python ${{ matrix.python-version }} on ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # Don't bail out on the first failing matrix cell — we want to
+      # see whether a failure is platform-specific or version-specific.
+      fail-fast: false
+      matrix:
+        # POSIX-only: the C++ kernel uses OpenMP via the system compiler,
+        # which is messy on Windows. Windows support is intentionally out
+        # of scope.
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # setuptools_scm reads tag history to derive the version;
+          # shallow clones come back as 0.0.0.
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install OpenMP runtime (macOS)
+        # AppleClang doesn't ship libomp; without this the find_package
+        # call in CMakeLists.txt fails to detect OpenMP and the kernel
+        # degrades to single-threaded.
+        if: runner.os == 'macOS'
+        run: brew install libomp
+
+      - name: Install package and test deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run pytest
+        run: pytest -v --cov=colstore --cov-report=term

colstore-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,171 @@
+name: Release
+
+# Triggered when a GitHub Release is published. We use `published`
+# rather than `created` so drafting a release in the GitHub UI doesn't
+# fire the publish step prematurely — the release has to actually go
+# live.
+#
+# Also exposes a manual `workflow_dispatch` button so maintainers can
+# build (but not publish) on demand for verification.
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false  # never cancel an in-progress publish
+
+permissions:
+  contents: read
+
+jobs:
+  # ---------------------------------------------------------------------
+  # 1. Sanity-gate: re-run the test suite on the release commit.
+  #
+  # CI on `main` already covers this for the normal flow, but a release
+  # can technically be tagged on any commit, so we re-verify before
+  # touching PyPI. We only run a single Python version here (latest
+  # stable) — the full matrix lives in ci.yml.
+  # ---------------------------------------------------------------------
+  test:
+    name: Sanity-test the release commit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # tags needed for setuptools_scm
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install package and test deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run pytest
+        run: pytest -v
+
+  # ---------------------------------------------------------------------
+  # 2a. Build the sdist on Linux.
+  #
+  # The sdist is platform-independent; one job is enough. It contains
+  # the Cython .pyx, C++ sources, and the CMake build description, so
+  # users on unsupported platforms can still build from source.
+  # ---------------------------------------------------------------------
+  build_sdist:
+    name: Build sdist
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # setuptools_scm
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build sdist
+        run: python -m build --sdist
+
+      - name: List built artifacts
+        run: ls -l dist/
+
+      - name: Upload sdist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-sdist
+          path: dist/*.tar.gz
+
+  # ---------------------------------------------------------------------
+  # 2b. Build per-platform wheels with cibuildwheel.
+  #
+  # colstore has a C++/Cython extension, so we can't ship a single
+  # py3-none-any wheel like a pure-Python package would. cibuildwheel
+  # runs the build inside manylinux containers for Linux and natively
+  # on macOS, producing wheels that pip can install on the same
+  # ABI/arch without a compiler on the user's machine.
+  # ---------------------------------------------------------------------
+  build_wheels:
+    name: Build wheels (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    needs: test
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # setuptools_scm
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.21
+        env:
+          # We support 3.10+; skip older interpreters, PyPy, 32-bit
+          # Linux, and musllinux (we don't test it).
+          CIBW_SKIP: "cp36-* cp37-* cp38-* cp39-* pp* *-manylinux_i686 *-musllinux*"
+          # The kernel's OpenMP path requires libomp on macOS;
+          # manylinux images already include it.
+          CIBW_BEFORE_ALL_MACOS: brew install libomp
+          # Quick smoke test on the built wheel before declaring it good.
+          CIBW_TEST_REQUIRES: pytest pandas
+          CIBW_TEST_COMMAND: pytest -q {project}/tests/test_format.py {project}/tests/test_factory.py
+
+      - name: List built artifacts
+        run: ls -l wheelhouse/
+
+      - name: Upload wheels artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-wheels-${{ matrix.os }}
+          path: ./wheelhouse/*.whl
+
+  # ---------------------------------------------------------------------
+  # 3. Publish to PyPI via Trusted Publishing (OIDC).
+  #
+  # This job ONLY runs on a real `release: published` event — the
+  # manual workflow_dispatch path stops at `build` so maintainers can
+  # download and inspect the artifacts without touching PyPI.
+  # ---------------------------------------------------------------------
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build_sdist, build_wheels]
+    if: github.event_name == 'release'
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/colstore
+
+    permissions:
+      id-token: write  # required for OIDC token used by Trusted Publishing
+
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          # All three uploaded artifacts (sdist + 2 wheel sets) get
+          # flattened into dist/ for the upload step.
+          merge-multiple: true
+
+      - name: List artifacts before upload
+        run: ls -l dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # No user/password — Trusted Publishing handles auth via OIDC.

colstore-0.1.0/.gitignore

@@ -0,0 +1,74 @@
+# Python
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+dist/
+wheelhouse/
+_skbuild/
+.eggs/
+*.egg-info/
+*.egg
+MANIFEST
+
+# Cython
+#   The .pyx is committed; the generated .cpp is a build artifact. Ignore
+#   that but keep our hand-written C++ kernel under src/cpp/.
+*.cpp
+!src/cpp/*.cpp
+cython_debug/
+
+# setuptools_scm — auto-generated at build time from git tags; never check in.
+src/colstore/_version.py
+
+# Installer / publisher
+pip-log.txt
+pip-delete-this-directory.txt
+.pypirc
+
+# Tests & coverage
+.pytest_cache/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+coverage.xml
+*.cover
+cover/
+.hypothesis/
+htmlcov/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Linting
+.ruff_cache/
+
+# Jupyter
+.ipynb_checkpoints
+profile_default/
+
+# Virtual environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+
+# Editor
+.vscode/
+.idea/
+*.swp
+
+# OS
+.DS_Store

colstore-0.1.0/CMakeLists.txt

@@ -0,0 +1,78 @@
+# CMake build for the colstore native extension.
+#
+# Builds a single Python extension module (``colstore._gather``) from a
+# Cython binding plus a C++ implementation. OpenMP is linked in when the
+# toolchain supports it (the kernel still works without OpenMP, just
+# single-threaded inside the inner loop).
+#
+# This file is driven by scikit-build-core via pyproject.toml. To build
+# manually without the Python packaging:
+#     cmake -S . -B build -G Ninja -DPython_EXECUTABLE=$(which python)
+#     cmake --build build
+
+cmake_minimum_required(VERSION 3.18)
+project(colstore LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+endif()
+
+# --- Dependencies --------------------------------------------------------
+find_package(Python COMPONENTS Interpreter Development.Module NumPy REQUIRED)
+find_package(OpenMP)
+
+# --- Cython preprocess ---------------------------------------------------
+# Invoke Cython at build time to translate _gather.pyx into _gather.cpp.
+# Done with a custom command rather than a CMake module so the user only
+# needs Cython installed via pip, not a CMake find module.
+set(CYTHON_PYX "${CMAKE_CURRENT_SOURCE_DIR}/src/cython/_gather.pyx")
+set(CYTHON_CPP "${CMAKE_CURRENT_BINARY_DIR}/_gather.cpp")
+
+add_custom_command(
+  OUTPUT ${CYTHON_CPP}
+  COMMAND ${Python_EXECUTABLE} -m cython
+          --cplus -3
+          -I "${CMAKE_CURRENT_SOURCE_DIR}/include"
+          -o "${CYTHON_CPP}"
+          "${CYTHON_PYX}"
+  DEPENDS ${CYTHON_PYX}
+          "${CMAKE_CURRENT_SOURCE_DIR}/include/colstore/gather.hpp"
+  COMMENT "Cythonizing _gather.pyx -> _gather.cpp"
+  VERBATIM)
+
+# --- Extension module ----------------------------------------------------
+Python_add_library(_gather MODULE WITH_SOABI
+  ${CYTHON_CPP}
+  src/cpp/gather.cpp
+)
+
+target_include_directories(_gather PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${Python_NumPy_INCLUDE_DIRS}
+)
+
+# Portable optimization flags. We deliberately do not pass -march=native so
+# wheels stay portable across CPUs; users who build from source for a
+# specific machine can override CMAKE_CXX_FLAGS_RELEASE.
+target_compile_options(_gather PRIVATE
+  $<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-O3 -ffast-math -funroll-loops>
+  $<$<CXX_COMPILER_ID:MSVC>:/O2 /fp:fast>
+)
+
+if(OpenMP_CXX_FOUND)
+  target_link_libraries(_gather PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+# Quiet a NumPy 2.x deprecation that fires harmlessly during Cython compile
+# of array-API users that don't opt into the new API yet.
+target_compile_definitions(_gather PRIVATE
+  NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION
+)
+
+# --- Install -------------------------------------------------------------
+# scikit-build-core copies this into the wheel under wheel.packages.
+install(TARGETS _gather DESTINATION colstore)

colstore-0.1.0/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.1
+Name: colstore
+Version: 0.1.0
+Summary: Memory-mapped columnar binary format for fast random-access I/O on structured arrays.
+Author-Email: Alkaid Cheng <alkaid.ccheng@gmail.com>
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: C++
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS
+Project-URL: Homepage, https://github.com/AlkaidCheng/colstore
+Project-URL: Issues, https://github.com/AlkaidCheng/colstore/issues
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.25
+Requires-Dist: psutil>=5.9
+Provides-Extra: pandas
+Requires-Dist: pandas>=1.5; extra == "pandas"
+Provides-Extra: progress
+Requires-Dist: tqdm>=4.60; extra == "progress"
+Provides-Extra: numba
+Requires-Dist: numba>=0.59; extra == "numba"
+Provides-Extra: all
+Requires-Dist: pandas>=1.5; extra == "all"
+Requires-Dist: tqdm>=4.60; extra == "all"
+Requires-Dist: numba>=0.59; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pandas>=1.5; extra == "dev"
+Requires-Dist: tqdm>=4.60; extra == "dev"
+Requires-Dist: numba>=0.59; extra == "dev"
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: black>=24.0; extra == "dev"
+Requires-Dist: mypy>=1.8; extra == "dev"
+Description-Content-Type: text/markdown
+
+# ColStore
+
+A memory-mapped columnar binary format for fast, memory-efficient I/O on
+structured arrays. `colstore` lets you write a tabular dataset to a single
+`.cstore` file once and then load arbitrary row/column subsets without
+materializing the rest. Internally, columns are stored back-to-back as raw
+NumPy bytes, reads use `np.memmap`, and fancy-index gathers run through a
+parallel C++ kernel (OpenMP + software prefetching) bound via Cython. Process
+memory stays bounded by the size of the output you ask for; the source file
+is never fully read into RAM.
+
+## Install
+
+```bash
+pip install colstore
+```
+
+Building from source needs a C++17 compiler and CMake ≥ 3.18. On macOS install
+`libomp` (`brew install libomp`) to get the parallel kernel; without it the
+build still succeeds but the kernel runs single-threaded.
+
+## Quick start
+
+```python
+from colstore import ColStore
+
+# Write and open in one call. `.cstore` is the canonical extension.
+ds = ColStore.from_dataframe(df, "data.cstore")
+
+# Indexing returns lazy views; no data is read yet.
+ds['price']                          # ColumnView
+ds[100:200]                          # TableView
+ds[100:200, 'price']                 # ColumnView
+ds[100:200, ['price', 'qty']]        # TableView
+ds[[1, 5, 9], ['price', 'qty']]      # TableView (fancy rows + cols)
+
+# Materialize through one of the to_* methods.
+ds['price'].to_array()                          # 1D ndarray
+ds[indices, ['price', 'qty']].to_dict()         # dict of 1D arrays
+ds[indices, ['price', 'qty']].to_record()       # structured ndarray
+ds[indices, ['price', 'qty']].to_dataframe()    # pandas DataFrame
+```
+
+## Writing from other sources
+
+```python
+from colstore import ColStore
+import numpy as np
+
+# From a dict of 1D arrays.
+ColStore.from_dict(
+    {"x": np.arange(100, dtype=np.float32), "y": np.arange(100, dtype=np.int64)},
+    "data.cstore",
+)
+
+# From a structured (record) array.
+records = np.empty(100, dtype=[("price", np.float32), ("qty", np.int32)])
+ColStore.from_records(records, "data.cstore")
+```
+
+Each factory returns an opened `ColStore` ready to read from.
+
+## Configuration
+
+```python
+from colstore import set_max_workers, set_default_madvise, set_default_backend
+
+set_max_workers(8)                # parallel gathers across columns
+set_default_madvise("sequential") # OS read-ahead hint for sorted-index reads
+set_default_backend("cpp")        # gather kernel: cpp | numpy | numba
+```
+
+## On-disk format
+
+```
+[magic 8B = b"CSTORE\x00\x01"]
+[manifest_len 8B (u64 little-endian)]
+[manifest_json]
+[zero-padding to 64-byte alignment]
+[column_0 raw bytes][column_1 raw bytes]...[column_n raw bytes]
+```
+
+The manifest is a small JSON object recording `format_version`, `n_rows`,
+and per-column `{name, dtype}`. Column dtypes are preserved byte-for-byte;
+columns are stored back-to-back with no per-row overhead.
+
+## Supported dtypes
+
+Fixed-size only: `float32`, `float64`, `int8/16/32/64`, `uint8/16/32/64`,
+`bool`. Object dtype (strings, Python objects) is rejected at write time —
+the design point is zero-copy random access, which requires a fixed stride.
+
+## Layout
+
+```
+colstore/
+├── pyproject.toml              # scikit-build-core build
+├── CMakeLists.txt              # Cython + C++ build
+├── include/colstore/
+│   └── gather.hpp              # public C++ header
+├── src/
+│   ├── cpp/gather.cpp          # OpenMP + prefetch kernel
+│   ├── cython/_gather.pyx      # dtype-dispatched binding
+│   └── colstore/               # Python package
+│       ├── __init__.py
+│       ├── config.py
+│       ├── format.py
+│       ├── kernels.py
+│       ├── view.py             # ColumnView + TableView
+│       └── store.py
+└── tests/                      # pytest suite
+```
+
+## License
+
+MIT.

colstore-0.1.0/README.md

@@ -0,0 +1,116 @@
+# ColStore
+
+A memory-mapped columnar binary format for fast, memory-efficient I/O on
+structured arrays. `colstore` lets you write a tabular dataset to a single
+`.cstore` file once and then load arbitrary row/column subsets without
+materializing the rest. Internally, columns are stored back-to-back as raw
+NumPy bytes, reads use `np.memmap`, and fancy-index gathers run through a
+parallel C++ kernel (OpenMP + software prefetching) bound via Cython. Process
+memory stays bounded by the size of the output you ask for; the source file
+is never fully read into RAM.
+
+## Install
+
+```bash
+pip install colstore
+```
+
+Building from source needs a C++17 compiler and CMake ≥ 3.18. On macOS install
+`libomp` (`brew install libomp`) to get the parallel kernel; without it the
+build still succeeds but the kernel runs single-threaded.
+
+## Quick start
+
+```python
+from colstore import ColStore
+
+# Write and open in one call. `.cstore` is the canonical extension.
+ds = ColStore.from_dataframe(df, "data.cstore")
+
+# Indexing returns lazy views; no data is read yet.
+ds['price']                          # ColumnView
+ds[100:200]                          # TableView
+ds[100:200, 'price']                 # ColumnView
+ds[100:200, ['price', 'qty']]        # TableView
+ds[[1, 5, 9], ['price', 'qty']]      # TableView (fancy rows + cols)
+
+# Materialize through one of the to_* methods.
+ds['price'].to_array()                          # 1D ndarray
+ds[indices, ['price', 'qty']].to_dict()         # dict of 1D arrays
+ds[indices, ['price', 'qty']].to_record()       # structured ndarray
+ds[indices, ['price', 'qty']].to_dataframe()    # pandas DataFrame
+```
+
+## Writing from other sources
+
+```python
+from colstore import ColStore
+import numpy as np
+
+# From a dict of 1D arrays.
+ColStore.from_dict(
+    {"x": np.arange(100, dtype=np.float32), "y": np.arange(100, dtype=np.int64)},
+    "data.cstore",
+)
+
+# From a structured (record) array.
+records = np.empty(100, dtype=[("price", np.float32), ("qty", np.int32)])
+ColStore.from_records(records, "data.cstore")
+```
+
+Each factory returns an opened `ColStore` ready to read from.
+
+## Configuration
+
+```python
+from colstore import set_max_workers, set_default_madvise, set_default_backend
+
+set_max_workers(8)                # parallel gathers across columns
+set_default_madvise("sequential") # OS read-ahead hint for sorted-index reads
+set_default_backend("cpp")        # gather kernel: cpp | numpy | numba
+```
+
+## On-disk format
+
+```
+[magic 8B = b"CSTORE\x00\x01"]
+[manifest_len 8B (u64 little-endian)]
+[manifest_json]
+[zero-padding to 64-byte alignment]
+[column_0 raw bytes][column_1 raw bytes]...[column_n raw bytes]
+```
+
+The manifest is a small JSON object recording `format_version`, `n_rows`,
+and per-column `{name, dtype}`. Column dtypes are preserved byte-for-byte;
+columns are stored back-to-back with no per-row overhead.
+
+## Supported dtypes
+
+Fixed-size only: `float32`, `float64`, `int8/16/32/64`, `uint8/16/32/64`,
+`bool`. Object dtype (strings, Python objects) is rejected at write time —
+the design point is zero-copy random access, which requires a fixed stride.
+
+## Layout
+
+```
+colstore/
+├── pyproject.toml              # scikit-build-core build
+├── CMakeLists.txt              # Cython + C++ build
+├── include/colstore/
+│   └── gather.hpp              # public C++ header
+├── src/
+│   ├── cpp/gather.cpp          # OpenMP + prefetch kernel
+│   ├── cython/_gather.pyx      # dtype-dispatched binding
+│   └── colstore/               # Python package
+│       ├── __init__.py
+│       ├── config.py
+│       ├── format.py
+│       ├── kernels.py
+│       ├── view.py             # ColumnView + TableView
+│       └── store.py
+└── tests/                      # pytest suite
+```
+
+## License
+
+MIT.