pysciqlop-cache 0.1.0__tar.gz

pysciqlop_cache-0.1.0/.bumpversion.cfg

@@ -0,0 +1,13 @@
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+
+[bumpversion:file:version.txt]
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
+search = {current_version}
+replace = {new_version}
+
+[bumpversion:file:docs/conf.py]
+search = version = '{current_version}'
+replace = version = '{new_version}'

pysciqlop_cache-0.1.0/.clang-format

@@ -0,0 +1,32 @@
+---
+# See http://clang.llvm.org/docs/ClangFormatStyleOptions.html for a definition 
+# of the options
+Language: Cpp
+BasedOnStyle:  WebKit
+
+# Line length
+ColumnLimit: 100
+
+# Indent with 4 spaces
+IndentWidth: 4
+AccessModifierOffset: -4 # -IndentWidth
+ConstructorInitializerIndentWidth: 8 # 2 * IndentWidth
+
+BreakBeforeBraces: Allman
+SeparateDefinitionBlocks: Always
+EmptyLineBeforeAccessModifier: Always
+
+AllowShortFunctionsOnASingleLine: Inline
+AlwaysBreakTemplateDeclarations: true
+AlwaysBreakBeforeMultilineStrings: true
+BreakBeforeBinaryOperators: true
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+IndentCaseLabels: true
+MaxEmptyLinesToKeep: 2
+Standard: Cpp11
+UseTab: Never
+
+ContinuationIndentWidth: 4
+AlignAfterOpenBracket: true
+LambdaBodyIndentation: Signature
+

pysciqlop_cache-0.1.0/.github/workflows/CI.yml

@@ -0,0 +1,213 @@
+name: GH Actions
+
+on:
+  release:
+    types: [published]
+  push:
+  pull_request:
+
+# Auto-cancel superseded runs of the same branch/PR. Releases never
+# cancel each other (the tag ref is unique).
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'release' && github.ref || (github.head_ref || github.ref_name) }}
+  cancel-in-progress: ${{ github.event_name != 'release' }}
+
+jobs:
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build sdist
+        run: pipx run build --sdist
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  build_wheels:
+    name: ${{ matrix.os }} ${{ matrix.CIBW_ARCHS }} wheels
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            CIBW_ENVIRONMENT: ''
+            CIBW_ARCHS: "x86_64"
+          - os: ubuntu-24.04-arm
+            CIBW_ENVIRONMENT: ''
+            CIBW_ARCHS: "aarch64"
+          - os: windows-latest
+            CIBW_ENVIRONMENT: ''
+            CIBW_ARCHS: "AMD64"
+          - os: macos-15-intel
+            CIBW_ENVIRONMENT: >
+              MACOSX_DEPLOYMENT_TARGET='13.0'
+            CIBW_ARCHS: "x86_64"
+          - os: macos-14 # Apple Silicon
+            CIBW_ENVIRONMENT: >
+              MACOSX_DEPLOYMENT_TARGET='13.0'
+            CIBW_ARCHS: "arm64"
+    env:
+      CIBW_ENVIRONMENT: ${{ matrix.CIBW_ENVIRONMENT }}
+      CIBW_SKIP: "*-win32 *i686"
+      CIBW_ARCHS: ${{ matrix.CIBW_ARCHS }}
+      CIBW_PRERELEASE_PYTHONS: "True"
+      # see https://github.com/matplotlib/matplotlib/pull/28687/files#diff-504e739c530d50b780e9a09ae8fd0c3ea8258ea7553be06afb06f03f95b1ee0aR103
+      CIBW_CONFIG_SETTINGS_WINDOWS: >-
+        setup-args="--vsenv"
+        setup-args="-Db_vscrt=mt"
+        setup-args="-Dcpp_link_args=['ucrt.lib','vcruntime.lib','/nodefaultlib:libucrt.lib','/nodefaultlib:libvcruntime.lib']"
+    steps:
+      - uses: actions/checkout@v4
+
+      # ccache for Linux wheels: cibuildwheel runs the build inside the
+      # manylinux/musllinux container, so we mount a host-side ccache dir
+      # into it (CIBW_CONTAINER_ENGINE), install ccache inside the
+      # container (CIBW_BEFORE_ALL_LINUX), and point CC/CXX at it. The
+      # cache mostly hits on stable subprojects (sqlite3.c is the big
+      # one — ~60s per build) across the 6 Python builds for one platform
+      # and across runs that don't bump subproject versions.
+      - name: Restore ccache (Linux)
+        if: runner.os == 'Linux'
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.ccache
+          key: ccache-${{ matrix.os }}-${{ matrix.CIBW_ARCHS }}-${{ hashFiles('subprojects/*.wrap', 'meson.build') }}
+          restore-keys: |
+            ccache-${{ matrix.os }}-${{ matrix.CIBW_ARCHS }}-
+
+      - name: Configure ccache for cibuildwheel (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          mkdir -p ${{ github.workspace }}/.ccache
+          chmod -R 777 ${{ github.workspace }}/.ccache
+          # Try every package manager / repo combo we might land in:
+          #   musllinux (Alpine: apk), manylinux_2_28 (AlmaLinux 8: PowerTools),
+          #   manylinux_2_34 (AlmaLinux 9: CRB), manylinux2014 (CentOS 7: EPEL).
+          INSTALL_CCACHE='( command -v apk && apk add --no-cache ccache ) || ( dnf install -y --enablerepo=powertools ccache 2>/dev/null ) || ( dnf install -y --enablerepo=crb ccache 2>/dev/null ) || ( yum install -y epel-release && yum install -y ccache )'
+          echo "CIBW_BEFORE_ALL_LINUX=$INSTALL_CCACHE" >> $GITHUB_ENV
+          echo 'CIBW_ENVIRONMENT_LINUX=CCACHE_DIR=/host_ccache CCACHE_MAXSIZE=500M CC="ccache gcc" CXX="ccache g++"' >> $GITHUB_ENV
+          echo 'CIBW_CONTAINER_ENGINE=docker; create_args: --volume=${{ github.workspace }}/.ccache:/host_ccache' >> $GITHUB_ENV
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.1.4
+
+      - name: Show ccache size (Linux)
+        if: runner.os == 'Linux' && always()
+        run: du -sh ${{ github.workspace }}/.ccache 2>/dev/null || true
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
+  test_wheels:
+    needs: [build_wheels]
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-15-intel, macos-14, windows-latest, ubuntu-latest, ubuntu-24.04-arm]
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+          #architecture: x64
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-wheels-*
+          path: dist
+          merge-multiple: true
+      - name: install wheel (Unix)
+        if: runner.os != 'Windows'
+        run: |
+          pip install numpy
+          pip install --no-index --find-links $GITHUB_WORKSPACE/dist pysciqlop-cache
+      - name: install wheel (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          pip install numpy
+          pip install --no-index --find-links $env:GITHUB_WORKSPACE\dist pysciqlop-cache
+      - uses: actions/checkout@v4
+      - name: run tests
+        # Two interacting issues, fixed together:
+        #
+        # 1. multiprocessing.Pool workers under spawn (macOS/Windows always)
+        #    or forkserver (Linux Python 3.14+ default) re-import the
+        #    function's module by name. pytest assigns name
+        #    'tests.python.test_concurrency_bugs', so workers need to be able
+        #    to `import tests`. Add repo root to PYTHONPATH for that.
+        #
+        # 2. But the source `pysciqlop_cache/` directory is at the repo root
+        #    and shadows the installed wheel: its __init__.py does
+        #    `from ._pysciqlop_cache import …` and the compiled .so only
+        #    exists in site-packages. Remove the source dir from the
+        #    workspace before running tests so the installed wheel wins.
+        #    (Cross-platform via Python's shutil.)
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+        run: |
+          pip install pytest pytest-timeout ddt requests hypothesis diskcache
+          python -c "import shutil; shutil.rmtree('pysciqlop_cache', ignore_errors=True)"
+          # --timeout=300 (5 min) so a hang fails fast with a stack trace
+          # instead of running until the GitHub job timeout. Real tests on
+          # Windows can be slow (cross-process SQLite + fcntl emulation) but
+          # 5 min is a generous ceiling — if anything exceeds it, that's a
+          # bug to investigate, not a number to bump.
+          # --timeout-method=thread so threaded/multiproc tests can be killed
+          # cleanly on Windows (the default 'signal' method needs SIGALRM
+          # which Windows doesn't have).
+          python -m pytest --import-mode=importlib --tb=short --disable-warnings -v --timeout=300 --timeout-method=thread tests/python --ignore=tests/python/test_torture.py --ignore=tests/python/test_hypothesis.py
+
+  upload_pypi:
+    needs: [build_sdist, build_wheels, test_wheels]
+    runs-on: ubuntu-latest
+    # upload to PyPI only on github releases
+    if: github.event_name == 'release' && github.event.action == 'published' && github.repository_owner == 'SciQLop'
+    # PyPI Trusted Publisher (OIDC). The `pypi` environment matches the
+    # publisher config registered on https://pypi.org/manage/project/PySciQLop_cache/
+    # — no API token needed; the action exchanges the OIDC ID token from
+    # GitHub for a short-lived PyPI upload token.
+    environment:
+      name: pypi
+      url: https://pypi.org/p/PySciQLop_cache
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true
+
+  upload_test_pypi:
+    needs: [build_sdist, build_wheels, test_wheels]
+    runs-on: ubuntu-latest
+    # upload to test PyPI on github pushes
+    if: github.event_name == 'push' && github.repository_owner == 'SciQLop'
+    # TestPyPI Trusted Publisher (OIDC). Matches the publisher config on
+    # https://test.pypi.org/manage/project/PySciQLop_cache/ via the
+    # `testpypi` environment.
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/PySciQLop_cache
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true

pysciqlop_cache-0.1.0/.github/workflows/tests-with-coverage.yml

@@ -0,0 +1,37 @@
+name: Tests Linux with coverage
+
+on: [push]
+
+jobs:
+  build:
+    name: build an tests with coverage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y python3-pip python3-dev lcov g++ meson ninja-build git catch2
+          pip install --upgrade meson ninja numpy meson-python>=0.14.0 build wheel ddt requests
+      - name: Configure with meson
+        run: meson -Db_coverage=true -Dwith_tests=true . build
+      - name: Build (meson)
+        run: ninja -C build
+      - name: Run tests (meson)
+        run: ninja test -C build
+      - name: Generate Coverage report
+        run: |
+          lcov --capture  --ignore-errors=inconsistent,mismatch --directory . --output-file coverage.info
+          lcov --remove coverage.info '/usr/*' --ignore-errors=unused --output-file coverage.info
+          lcov --remove coverage.info '*/catch*' --ignore-errors=unused --output-file coverage.info
+          lcov --list coverage.info
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.info
+          flags: unittests
+          name: codecov-cdfpp
+          fail_ci_if_error: true

pysciqlop_cache-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+build/
+subprojects/*
+!subprojects/*.wrap
+*.kdev4
+CMakeLists.txt.user
+.cache/*
+.cmake/*
+*.user
+*.user.*
+*.ipynb_check*
+*__pycache__*
+version.txt
+docs/_build/*
+docs/generated/*
+dist/*
+tmp*
+.vscode/*

pysciqlop_cache-0.1.0/CLAUDE.md

@@ -0,0 +1,88 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Sciqlop-cache is a C++20 caching library with Python bindings. It uses SQLite for metadata and a hybrid storage strategy: values ≤ 8KB stored as BLOBs in SQLite, larger values stored as files on disk. Thread-safe via per-instance database connections and WAL mode.
+
+## Build Commands
+
+```bash
+# Configure with tests
+meson setup build -Dwith_tests=true
+
+# Build
+meson compile -C build
+
+# Run all tests
+meson test -C build
+
+# Run a single test suite
+meson test -C build sciqlop-cache:basic
+
+# Available test suites:
+#   C++ (always-on):  sciqlop-cache:basic, sciqlop-cache:basic_index,
+#                     sciqlop-cache:database, sciqlop-cache:intermediate,
+#                     sciqlop-cache:multithreads, sciqlop-cache:fanout,
+#                     sciqlop-cache:check
+#   C++ (with_torture_tests=true):
+#                     sciqlop-cache:torture, sciqlop-cache:concurrency_bugs
+#   Python (always-on if Python wrapper enabled):
+#                     sciqlop-cache:test_python_interface,
+#                     sciqlop-cache:test_serializers,
+#                     sciqlop-cache:test_python_multiprocess,
+#                     sciqlop-cache:test_perf_vs_diskcache,
+#                     sciqlop-cache:test_concurrency_bugs
+#   Python (with_torture_tests=true):
+#                     sciqlop-cache:test_torture, sciqlop-cache:test_hypothesis
+
+# Build Python wheel
+pip install meson-python numpy && python -m build --wheel
+```
+
+### Meson Options
+
+- `with_tests` (false) — build C++ tests
+- `with_benchmarks` (false) — build benchmarks
+- `disable_python_wrapper` (false) — skip Python bindings
+- `tracy_enable` (false) — enable Tracy profiling
+
+## Architecture
+
+**Core layer** (`include/sciqlop_cache/`):
+
+- `store.hpp` — `_Store<Storage, Policies...>` template class, the main engine. Uses zero-cost policy-based design: policies are inherited as mixins, SQL and behavior composed via `if constexpr` + fold expressions. No virtual dispatch.
+- `policies.hpp` — Policy structs: `WithExpiration`, `WithEviction`, `WithTags`, `WithStats`. Each contributes schema columns, WHERE clause fragments, and runtime hooks. `has_policy_v` trait for compile-time branching.
+- `sciqlop_cache.hpp` — Type aliases: `Cache = _Store<DiskStorage, WithExpiration, WithEviction, WithTags, WithStats>`, `Index = _Store<DiskStorage>`, `FanoutCache = FanoutStore<Cache>`, `FanoutIndex = FanoutStore<Index>`.
+- `fanout_store.hpp` — `FanoutStore<StoreType>` template. Shards keys across N independent `_Store` instances via `hash(key) % shard_count` for write concurrency. Per-key ops dispatch to one shard; cross-shard ops aggregate.
+- `database.hpp` — SQLite wrapper: `Database`, `CompiledStatement`, `BindedCompiledStatement`, `Transaction`. Uses SQLITE_NOMUTEX with manual transaction control. `~Transaction()` rolls back if `commit()` was never called (RAII rollback-by-default).
+- `disk_storage.hpp` — `DiskStorage` class. UUID-based two-level directory hierarchy for file storage.
+- `utils/concepts.hpp` — C++20 concepts: `DurationConcept`, `TimePoint`, `Bytes`
+- `utils/buffer.hpp` — Polymorphic buffer with memory-mapped file support
+- `utils/time.hpp` — Epoch/TimePoint conversions for SQLite
+
+**Python bindings** (`pysciqlop_cache/`): nanobind-based, exposes `Cache`, `Index`, `FanoutCache`, and `FanoutIndex` with dict-like interface and `Buffer` with `.memoryview()`.
+
+**Tests** (`tests/`): Catch2 BDD-style. Seven always-on C++ suites (basic, basic_index, database, intermediate, multithreads, fanout, check) plus Python tests. Two more suites (`torture`, `concurrency_bugs`) plus the Python `test_torture` and `test_hypothesis` are gated by `-Dwith_torture_tests=true` (designed for sanitizer / long-running runs).
+
+## Dependencies
+
+All vendored in `subprojects/`: SQLite amalgamation, fmt, nanobind, Catch2, stduuid, cpp_utils, tracy, robin-map, hedley.
+
+## Key Design Decisions
+
+- **Policy-based Store** — `_Store<Storage, Policies...>` composes features at compile time. `Cache` has expiration, LRU eviction, tags, and stats. `Index` is a bare key-value store with no overhead.
+- **FanoutStore** — `FanoutStore<StoreType>` shards keys across N independent stores (default 8) for write concurrency. `max_size` is per shard. `transact(key)` scoped to one shard. No cross-shard transactions.
+- Per-instance `Database` + `std::recursive_mutex` for thread safety
+- WAL mode + 600s busy_timeout for multi-process safety
+- **`_NestedTxn` (private RAII helper in `_Store`)** — every internal write path (`_set_impl`, `del`, `pop`, `incr`, `evict_tag`) wraps in a `BEGIN EXCLUSIVE` only at the outermost level (depth-counted via `_txn_depth`). Inner levels are no-ops. Both for cross-process atomicity (read-modify-write inside one txn) and to compose cleanly inside a user `transact()`.
+- **`TransactionGuard` is reentrant on the same thread** (depth-counted same as `_NestedTxn`). Nested `with cache.transact():` is supported; outer rollback discards inner work (no real SAVEPOINTs — same semantics as diskcache).
+- **BG checkpoint thread takes `_mtx`** around `_bg_evict` + `_resync_counters` so it can't clobber atomic counters mid-update on the user-thread side.
+- `size()` and `count()` use in-memory atomic counters (O(1)) for types without expiration; `count()` queries DB when expiration filtering needed (deliberate — see `tier2_perf_decisions.md` in the auto-memory)
+- Expiration handled at query time via SQL (`WHERE expire IS NULL OR expire > unixepoch('now')`) — only present in types with `WithExpiration`
+- No-expiry default: `set()`/`add()` without expire stores NULL (never expires)
+- LRU eviction: `max_size` in bytes (0 = unlimited, default). Background thread evicts using monotonic access counter — only present with `WithEviction`
+- Tags: optional `tag` parameter on `set()`/`add()`, indexed for fast `evict_tag()` bulk removal — only present with `WithTags`
+- C++20 concepts enforce type safety at compile time
+- `requires` clauses on methods ensure policy-specific API (e.g. `touch()`, `evict()`) is only available on types that support it

pysciqlop_cache-0.1.0/COPYING

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Laboratory of Plasma Physics, Ecole Polytechnique, CNRS, Sorbonne Université, Université Paris-Saclay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.