pydirs 0.1.1__tar.gz

pydirs-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,133 @@
+name: CI
+
+# Runs only when a GitHub Release is created, not on every push or pull request.
+on:
+  release:
+    types: [created]
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  lint:
+    name: Lint and Rust tests (no Python feature)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Check formatting
+        run: cargo fmt --all -- --check
+
+      - name: Clippy (warnings are errors)
+        run: cargo clippy --no-default-features --all-targets -- -D warnings
+
+      - name: Rust tests
+        run: cargo test --no-default-features
+
+  test:
+    name: Python ${{ matrix.python }} on ${{ matrix.os }}
+    needs: lint
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Install build and test tools
+        run: pip install maturin "pytest>=7" pytest-asyncio
+
+      # `maturin develop` needs a venv and mis-detects the interpreter inside one on Windows,
+      # so build a wheel against the setup-python interpreter and install it.
+      - name: Build wheel (release)
+        run: maturin build --release --out dist
+
+      - name: Install wheel
+        run: pip install --no-index --find-links dist pydirs
+
+      # No version pin: pip resolves a pydantic release that supports each Python in the matrix.
+      - name: Install pydantic (optional extra, exercised by tests)
+        run: pip install pydantic
+
+      - name: Run Python tests (leak tests run in the leaks job)
+        run: pytest -m "not leak"
+
+  leaks:
+    name: Memory-leak tests (Linux)
+    needs: lint
+    runs-on: ubuntu-latest
+    env:
+      # Leak tests read this to scale iteration counts down under valgrind.
+      PYDIRS_VALGRIND: "1"
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Install valgrind and jq
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind jq
+
+      - name: Build Rust test binaries
+        run: |
+          cargo test --no-default-features --no-run --message-format=json > cargo-test-build.json
+          # Both the lib unit-test binary and the tests/leaks.rs binary must build.
+          TARGETS='["_pydirs", "leaks"]'
+          jq -r --argjson targets "$TARGETS" '
+            select(.reason == "compiler-artifact" and .profile.test == true and .executable != null)
+            | select(.target.name as $n | $targets | index($n))
+            | .executable' cargo-test-build.json > test-binaries.txt
+          echo "Test binaries to check:"
+          cat test-binaries.txt
+          if [ ! -s test-binaries.txt ]; then
+            echo "::error::No lib test binary found in cargo JSON output."
+            exit 1
+          fi
+
+      - name: Run Rust tests under valgrind
+        run: |
+          while read -r bin; do
+            echo "::group::valgrind $bin"
+            valgrind --leak-check=full --errors-for-leak-kinds=definite \
+              --error-exitcode=1 "$bin" --test-threads=1
+            echo "::endgroup::"
+          done < test-binaries.txt
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build and install extension (release)
+        run: |
+          python -m venv .venv
+          echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
+          echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
+          . .venv/bin/activate
+          pip install maturin "pytest>=7" pytest-asyncio pydantic
+          maturin develop --release
+
+      - name: Run Python leak tests
+        run: |
+          pytest -m leak -v

pydirs-0.1.1/.github/workflows/release.yml

@@ -0,0 +1,161 @@
+name: Release
+
+# Only runs when a GitHub Release is published. Pushes, tags and manual
+# dispatch do NOT trigger this workflow.
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  check-version:
+    name: Verify tag matches Cargo.toml version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compare release tag with package version
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+        run: |
+          TAG_VERSION="${TAG#v}"
+          CARGO_VERSION=$(grep -m1 '^version' Cargo.toml | sed -E 's/^version *= *"([^"]+)".*/\1/')
+          echo "Release tag: ${TAG} (version ${TAG_VERSION})"
+          echo "Cargo.toml version: ${CARGO_VERSION}"
+          if [ "${TAG_VERSION}" != "${CARGO_VERSION}" ]; then
+            echo "::error::Release tag '${TAG}' does not match Cargo.toml version '${CARGO_VERSION}'. Bump Cargo.toml and re-release."
+            exit 1
+          fi
+
+  build-wheels:
+    name: Wheels ${{ matrix.name }}
+    needs: check-version
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux-x86_64
+            runs-on: ubuntu-latest
+            target: x86_64
+            manylinux: auto
+          - name: linux-aarch64
+            runs-on: ubuntu-latest
+            target: aarch64
+            manylinux: auto
+          - name: musllinux-x86_64
+            runs-on: ubuntu-latest
+            target: x86_64
+            manylinux: musllinux_1_2
+          - name: musllinux-aarch64
+            runs-on: ubuntu-latest
+            target: aarch64
+            manylinux: musllinux_1_2
+          - name: macos-x86_64
+            runs-on: macos-latest
+            target: x86_64
+            manylinux: ""
+          - name: macos-aarch64
+            runs-on: macos-latest
+            target: aarch64
+            manylinux: ""
+          - name: windows-x64
+            runs-on: windows-latest
+            target: x64
+            manylinux: ""
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.manylinux }}
+          args: --release --out dist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-wheels-${{ matrix.name }}
+          path: dist/*.whl
+          if-no-files-found: error
+
+  build-sdist:
+    name: Source distribution
+    needs: check-version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-sdist
+          path: dist/*.tar.gz
+          if-no-files-found: error
+
+  smoke-test:
+    name: Smoke test on Python ${{ matrix.python }}
+    needs: [build-wheels, build-sdist]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python: ["3.9", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+
+      - name: Install wheel from artifacts (no index)
+        run: |
+          pip install --no-index --find-links dist pydirs
+          pip list | grep -i pydirs
+
+      - name: Import and round-trip a value
+        working-directory: /tmp
+        run: |
+          python -c "
+          import pydirs
+          from pydirs import Cache
+          print('pydirs', pydirs.__version__)
+          with Cache() as c:
+              assert c.set('smoke', 'ok') is True
+              assert c.get('smoke') == b'ok'
+          print('smoke test passed')
+          "
+
+  publish:
+    name: Publish to PyPI
+    needs: [check-version, build-wheels, build-sdist, smoke-test]
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed for PyPI trusted publishing (OIDC). Also harmless with an API token.
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+
+      - name: List artifacts to publish
+        run: ls -la dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          skip-existing: true

pydirs-0.1.1/.gitignore

@@ -0,0 +1,31 @@
+# Rust
+/target/
+**/*.rs.bk
+Cargo.lock.bak
+
+# Python
+__pycache__/
+*.py[cod]
+*.so
+*.pyd
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# maturin
+*.whl
+
+# Editors and OS
+.idea/
+.vscode/
+*.swp
+.DS_Store
+
+PLAN.md
+.claude/

pydirs-0.1.1/CHANGELOG.md

@@ -0,0 +1,112 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.1] - 2026-10-15
+
+### Changed
+
+- Renamed the project to `pydirs` (PyPI distribution, import name, native module
+  `_pydirs`, `pydirs-server` command, `PYDIRS_PASSWORD`), because `tinycache-rs`
+  could not be published under the old name.
+
+### Fixed
+
+- Reopening a data directory right after dropping a store no longer fails with a
+  spurious "locked" error while the sweeper thread releases its last reference.
+- `Server.stop` keeps waking the accept thread until it exits, so shutdown cannot hang.
+- Release and CI workflows: `id-token: write` for PyPI publishing, wheel-based CI
+  install that works on Windows.
+
+## [0.1.0] - 2026-10-14
+
+### Added
+
+- Rust core: sharded in-memory store (64 shards by default, `parking_lot` locks)
+  with string, hash, list and set values.
+- Key expiry with `ex` / `px` on `set`, `expire`, `pexpire`, `persist`, `ttl`
+  and `pttl`. Lazy expiry on read plus a background sweeper thread.
+- Optional memory limit (`max_memory`) with `noeviction`, `allkeys-lru` and
+  `volatile-lru` policies using sampled LRU.
+- Python package `pydirs` (distribution `pydirs`) built with PyO3 and
+  maturin against the `abi3-py39` stable ABI, supporting Python 3.9 through 3.14.
+- `Cache` class with dict-style access, context-manager support and a `decode`
+  option for `str` reads.
+- `memoize(ttl=...)` decorator on `Cache` instances.
+- JSON support using the standard library `json` module only (no pickle):
+  `set_json`, `get_json`, `mget_json`, `mset_json`, `hset_json`, `hget_json`,
+  `lpush_json`, `rpush_json` and `lrange_json`.
+- `Cache(serializer="json")` routes `set`, `get`, `mget`, `mset`, dict-style
+  access and `memoize` through JSON, so dicts, lists and other JSON-compatible
+  values round-trip. The default remains `serializer="raw"`.
+- Rejection of `NaN` and `Infinity` so stored values are strictly valid JSON.
+
+### Changed
+
+- The core stores each string key and value in one allocation, with 40-byte table
+  entries backed by `hashbrown`. Memory accounting rounds to 16-byte allocation
+  granules and counts shard bucket arrays, so `used_memory` tracks RSS growth more
+  closely. Constants are calibrated with `examples/mem_probe.rs`.
+- `Cache` and `Client` share one API mixin, so both support the same
+  serializers, JSON, pydantic, decoding and `memoize`.
+- Optional pydantic support (`pip install "pydirs[pydantic]"`, pydantic
+  v2 only): `set_model`, `get_model`, `mget_model`, `hset_model` and
+  `hget_model`. The package works without pydantic, which is imported lazily.
+- `memoize` returns real pydantic model instances when the return annotation
+  names a pydantic type. It accepts a `model=` override, supports async
+  functions, keeps a cached `None` distinct from a miss, and keys
+  pydantic-model arguments by `model_dump_json()`.
+- `Server` class: RESP2 TCP server compatible with `redis-py`. Supports
+  strings, keys, hashes, lists, sets, `PING`, `ECHO`, `SETEX`, `DECRBY`,
+  `FLUSHDB`, `SELECT` (database 0), `COMMAND`, `AUTH`, `INFO`, `SAVE`, `BGSAVE`,
+  `LASTSAVE` and `QUIT`. `Server` is a context manager, and `port` reports the
+  port actually bound.
+- Password authentication: `Server(password=...)` and `Client(password=...)`.
+  Until a connection sends a correct `AUTH`, only `AUTH` and `QUIT` are accepted,
+  and other commands get `NOAUTH`. A wrong password gets `WRONGPASS`.
+- Persistence: `Cache(persist_dir=..., fsync="everysec"|"always"|"never",
+  snapshot_interval=300)`, with `save()`, `persistent` and `persist_stats`. The
+  design is a snapshot plus a write-ahead log in one directory, with a `LOCK`
+  file. `close()` (and leaving a `with` block) writes a final snapshot. See
+  `docs/persistence.md` for the guarantees and their limits.
+- Persistence exceptions: `CacheLockedError` (a subclass of `OSError`) when
+  another live process holds the directory, and `CorruptDataError` (a subclass of
+  `ValueError`) when the snapshot fails its checksum or structure check.
+- `AuthenticationError` (a subclass of `ConnectionError`) for missing or wrong
+  passwords on a `Client`.
+- `Client` administration: `info()`, `save()`, `bgsave()` and `lastsave()`.
+  `used_memory` on a `Client` reads `INFO`.
+- `Client`: thread-safe, pure-Python RESP2 client for a running server, with the
+  same API as `Cache`. It has a connection pool (`pool_size`), per-socket
+  `timeout`, `AUTH` via `password`, and retries an idempotent command once after
+  a dropped connection. Non-idempotent commands are never retried.
+- `Pipeline` (`client.pipeline()`) sends several commands in one write. Results
+  are in `p.results` and `execute()`.
+- Serializer protocol: `Cache(serializer=...)` accepts `"raw"`, `"json"`, or any
+  object with `dumps` and `loads` methods. `JsonSerializer` and
+  `PydanticSerializer` are provided.
+- Command-line server: `pydirs-server` and `python -m pydirs`, with
+  `--host`, `--port`, `--max-memory`, `--policy`, `--shards`, `--dir`, `--fsync`,
+  `--snapshot-interval`, `--requirepass` (or `PYDIRS_PASSWORD`),
+  `--allow-insecure` and `--version`. It refuses non-loopback binds without a
+  password (exit code 2). A locked or corrupt data directory exits with code 1.
+  On SIGINT or SIGTERM it stops and, with `--dir`, writes a final snapshot.
+- Memory-leak tests: a Rust counting-allocator suite (`tests/leaks.rs`), Python
+  RSS and fd tests under the `leak` marker, and a Linux `leaks` CI job running
+  the Rust tests under valgrind.
+- The GIL is released around every core call so Python threads run in parallel.
+- Prebuilt wheels for Linux (manylinux and musllinux, x86_64 and aarch64),
+  macOS (x86_64 and arm64) and Windows (x64), plus a source distribution.
+- CI workflow: Rust lint and tests, and a Python test matrix over Ubuntu, macOS
+  and Windows.
+- Release workflow: publishes to PyPI when a GitHub Release is published, after
+  verifying that the tag matches the `Cargo.toml` version.
+
+[Unreleased]: https://github.com/4ursmile/pydirs/compare/v0.1.1...HEAD
+[0.1.1]: https://github.com/4ursmile/pydirs/releases/tag/v0.1.1
+[0.1.0]: https://github.com/4ursmile/pydirs/releases/tag/v0.1.0