pipnn 0.1.0__tar.gz

pipnn-0.1.0/.dockerignore

@@ -0,0 +1,12 @@
+target/
+.venv/
+.git/
+pages/
+*.pdf
+*.h5ad
+*.npy
+__pycache__/
+.pytest_cache/
+.ipynb_checkpoints/
+notebooks/
+docker/out/

pipnn-0.1.0/.github/workflows/ann-isorecall-x86.yml

@@ -0,0 +1,85 @@
+name: ANN iso-recall (x86, with pyglass)
+
+# Manual-only: builds the PiPNN extension on x86_64 Linux, installs pyglass
+# (glassppy — manylinux x86_64 / CPython 3.10 only, can't run on the arm64 dev
+# machine) alongside FAISS, and runs bench/bench_ann_isorecall.py to produce a
+# real PiPNN vs pyglass vs FAISS recall-QPS + build-time figure. Artifacts: the
+# png + json. Trigger: Actions tab → this workflow → Run, or `gh workflow run`.
+
+on:
+  workflow_dispatch:
+    inputs:
+      dataset:
+        description: "ANN-benchmark dataset"
+        type: choice
+        options: [sift, gist]
+        default: sift
+      n_sub:
+        description: "Subsample base to this many points (0 = full 1M)"
+        type: string
+        default: "200000"
+
+jobs:
+  isorecall:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Informational: pyglass is built from source for the runner's own arch
+      # (-march=native), so AVX2 is enough — no AVX-512 needed.
+      - name: Runner CPU SIMD flags
+        run: |
+          echo "avx2: $(grep -c avx2 /proc/cpuinfo || true)  avx512f: $(grep -c avx512f /proc/cpuinfo || true)"
+          grep -m1 -o 'model name.*' /proc/cpuinfo || true
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      # glassppy ships only as a CPython 3.10 manylinux x86_64 wheel.
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      # Cache the (large) ann-benchmarks hdf5 + recomputed ground truth.
+      - name: Cache dataset
+        uses: actions/cache@v4
+        with:
+          path: bench/data
+          key: ann-data-${{ inputs.dataset }}
+
+      - name: Build extension + install backends
+        run: |
+          python -m venv .venv
+          . .venv/bin/activate
+          python -m pip install --upgrade pip maturin
+          pip install numpy scipy scikit-learn matplotlib h5py faiss-cpu pybind11 setuptools wheel
+          maturin develop --release
+          # pyglass: build FROM SOURCE (installs as `glass`), not the PyPI wheel.
+          # The wheel is compiled for AVX-512 and SIGILLs on GitHub's AMD EPYC
+          # runners (no AVX-512); the source build uses `-march=native`, so it
+          # targets the runner's own AVX2 and runs. Non-fatal — the harness still
+          # emits a PiPNN-vs-FAISS curve if this fails.
+          git clone --depth 1 https://github.com/zilliztech/pyglass.git /tmp/pyglass || true
+          if [ -d /tmp/pyglass/python ]; then
+            # build in-place (no isolation): setup.py references ../glass, ../third_party
+            ( cd /tmp/pyglass/python && python setup.py bdist_wheel && pip install dist/*.whl ) \
+              || echo "::warning::pyglass source build failed; arm will be skipped"
+          fi
+          python -c "import glass; print('glass source build OK', getattr(glass,'__version__','?'))" \
+            || echo "::warning::pyglass ('glass') not importable; arm will be skipped"
+
+      - name: Run iso-recall harness
+        run: |
+          . .venv/bin/activate
+          # -u: unbuffered, so PiPNN/FAISS results reach the log even if the
+          # isolated pyglass child SIGILLs on an unsupported instruction set.
+          python -u bench/bench_ann_isorecall.py --dataset "${{ inputs.dataset }}" --n-sub "${{ inputs.n_sub }}"
+
+      - name: Upload results
+        if: always()  # PiPNN-vs-FAISS artifact is written before the pyglass arm
+        uses: actions/upload-artifact@v4
+        with:
+          name: ann-isorecall-${{ inputs.dataset }}
+          if-no-files-found: warn
+          path: |
+            bench/ann_isorecall_${{ inputs.dataset }}.png
+            bench/ann_isorecall_${{ inputs.dataset }}.json

pipnn-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Cargo tests (core crate)
+        run: cargo test -p pipnn-core --release
+
+      - name: Build extension + run Python tests
+        run: |
+          python -m venv .venv
+          . .venv/bin/activate
+          python -m pip install --upgrade pip maturin
+          pip install numpy scipy scikit-learn scanpy anndata igraph leidenalg pynndescent faiss-cpu pytest
+          maturin develop --release
+          pytest tests/ -q

pipnn-0.1.0/.github/workflows/pyglass-portable-wheels.yml

@@ -0,0 +1,62 @@
+name: pyglass-portable wheels
+
+# Builds portable pyglass wheels (AVX2 on x86, NEON on arm64) from
+# tools/pyglass-portable/portable.patch, across a multi-OS/arch matrix, and
+# uploads them as artifacts. Proves one wheel per arch runs on that whole arch —
+# unlike upstream's AVX-512-only `glassppy` wheel. Manual trigger.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  wheels:
+    name: ${{ matrix.label }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu-latest, label: "linux-x86_64 (AVX2)"}
+          - {os: macos-14,      label: "macos-arm64 (NEON)"}
+          - {os: macos-13,      label: "macos-x86_64 (AVX2)"}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: CPU SIMD flags (informational)
+        run: |
+          if [ "$(uname -s)" = "Linux" ]; then
+            echo "avx2: $(grep -c avx2 /proc/cpuinfo) avx512f: $(grep -c avx512f /proc/cpuinfo)"
+            grep -m1 -o 'model name.*' /proc/cpuinfo || true
+          else
+            sysctl -n machdep.cpu.brand_string 2>/dev/null || true
+            echo "arch: $(uname -m)"
+          fi
+
+      - name: Build portable wheel
+        run: |
+          python -m pip install --upgrade pip
+          PYTHON="$(which python)" bash tools/pyglass-portable/build_wheel.sh "$PWD/wheelhouse"
+
+      - name: Smoke-test the wheel (import + build + search)
+        run: |
+          python -m pip install numpy scikit-learn
+          python -m pip install --force-reinstall wheelhouse/glass-*.whl
+          python - <<'PY'
+          import numpy as np, glass, platform, time
+          b = np.ascontiguousarray(np.random.default_rng(0).normal(size=(20000, 96)).astype(np.float32))
+          i = glass.Index("HNSW", "L2", "FP32", 32, 200); g = i.build(b)
+          s = glass.Searcher(g, b, "L2", "FP32"); s.set_ef(64); s.optimize()
+          t = time.perf_counter(); ids, _ = s.batch_search(b[:500], 10); dt = time.perf_counter() - t
+          print(f"OK pyglass on {platform.machine()}: {500/dt:.0f} q/s")
+          PY
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: pyglass-portable-${{ matrix.os }}
+          path: wheelhouse/glass-*.whl
+          if-no-files-found: error

pipnn-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,107 @@
+name: Release to PyPI
+
+# Builds abi3 wheels (one per platform serves Python 3.9+) for Linux, macOS, and
+# Windows, plus an sdist, and publishes them to PyPI on a version tag via PyPI
+# Trusted Publishing (no API token stored). Tag a release: `git tag v0.1.0 &&
+# git push --tags`. workflow_dispatch runs the build only (no publish) as a dry run.
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build wheels (manylinux, abi3)
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: auto
+          args: --release --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.target }}
+          path: dist
+
+  macos:
+    runs-on: macos-14
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [aarch64, x86_64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build wheel (abi3)
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.target }}
+          path: dist
+
+  windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build wheel (abi3)
+        uses: PyO3/maturin-action@v1
+        with:
+          target: x64
+          args: --release --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-windows
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+  publish:
+    name: Publish to PyPI
+    needs: [linux, macos, windows, sdist]
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # required for PyPI Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist

pipnn-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Rust
+/target
+**/*.rs.bk
+
+# Python
+.venv/
+__pycache__/
+*.py[cod]
+*.so
+*.egg-info/
+.pytest_cache/
+.ipynb_checkpoints/
+
+# Paper (cite arXiv:2602.21247 instead of redistributing)
+PiPNN.pdf
+/pages/
+
+# Data
+*.h5ad
+*.npy
+# ANN-benchmark dataset cache (SIFT/GIST hdf5, ~0.5–3.6 GB each)
+bench/data/
+
+# Docker bench outputs
+docker/out/
+
+# OS
+.DS_Store
+# Tahoe-100M dataset cache (parquet shards + PCA embedding, multi-GB)
+bench/tahoe_cache/