straw-queue 0.1.0__tar.gz

straw_queue-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,69 @@
+name: tests
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    name: Rust tests and code checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install formatters
+        run: python -m pip install 'black>=24.8' 'isort>=5.13'
+      - name: Check formatting
+        run: |
+          cargo fmt --check
+          python -m black --check src tests examples tools verification
+          python -m isort --check-only src tests examples tools verification
+      - name: Lint native core and Python bindings
+        run: |
+          cargo clippy --locked --all-targets -- -D warnings
+          cargo clippy --locked --all-targets --features python -- -D warnings
+      - name: Run Rust tests
+        run: cargo test --locked
+
+  python:
+    name: Full suite / Python ${{ matrix.python }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install build and test dependencies
+        run: |
+          python -m pip install 'build>=1.2' 'twine>=5' 'numpy>=1.24' 'pytest>=8'
+          python -m pip install 'torch>=2.3' --index-url https://download.pytorch.org/whl/cpu
+      - name: Build the wheel
+        run: python -m build --wheel
+      - name: Check package metadata
+        run: python -m twine check dist/*
+      - name: Run all Python and protocol tests against the installed wheel
+        # Includes examples, multiprocess recovery, the ownership model, crash
+        # injection, a bounded I/O/GC benchmark and trace verification.
+        run: python tools/check_wheel.py dist/*.whl --tests --report wheel-check.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-report-py${{ matrix.python }}
+          path: wheel-check.json
+          if-no-files-found: warn

straw_queue-0.1.0/.github/workflows/wheels.yml

@@ -0,0 +1,149 @@
+name: wheels
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    tags: ['v*']
+permissions:
+  contents: read
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-tag: ['310', '311', '312', '313']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: PyO3/maturin-action@v1
+        with:
+          target: x86_64
+          # The action adds --manylinux; do not also pass --compatibility in args.
+          manylinux: '2_28'
+          args: --release --locked --strip --out dist --interpreter /opt/python/cp${{ matrix.python-tag }}-cp${{ matrix.python-tag }}/bin/python
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel-cp${{ matrix.python-tag }}-linux-x86_64
+          path: dist/*.whl
+          if-no-files-found: error
+  smoke:
+    name: Verify wheel / Python ${{ matrix.python }}
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python: '3.10'
+            tag: '310'
+          - python: '3.11'
+            tag: '311'
+          - python: '3.12'
+            tag: '312'
+          - python: '3.13'
+            tag: '313'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: wheel-cp${{ matrix.tag }}-linux-x86_64
+          path: dist
+      - run: python -m pip install 'numpy>=1.24' 'pytest>=8' 'twine>=5'
+      - run: python -m pip install 'torch>=2.3' --index-url https://download.pytorch.org/whl/cpu
+      - run: python -m twine check dist/*
+      - run: python tools/check_wheel.py dist/*.whl --tests --report wheel-check.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: wheel-check-cp${{ matrix.tag }}
+          path: wheel-check.json
+  sdist:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install 'build>=1.2' 'twine>=5' 'numpy>=1.24' 'pytest>=8'
+      - run: python -m pip install 'torch>=2.3' --index-url https://download.pytorch.org/whl/cpu
+      - run: python -m build --sdist
+      - run: python -m twine check dist/*
+      - run: python -m pip wheel --no-deps dist/*.tar.gz --wheel-dir rebuilt
+      - run: python -m twine check rebuilt/*
+      - run: python tools/check_wheel.py rebuilt/*.whl --tests --report wheel-check.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: source-distribution
+          path: dist/*.tar.gz
+          if-no-files-found: error
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: source-rebuild-check
+          path: wheel-check.json
+  publish:
+    name: Publish to PyPI
+    needs: [smoke, sdist]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment:
+      name: pypi
+      url: https://pypi.org/p/straw-queue
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Download tested wheels
+        uses: actions/download-artifact@v4
+        with:
+          pattern: wheel-cp*-linux-x86_64
+          merge-multiple: true
+          path: dist
+      - name: Download tested source distribution
+        uses: actions/download-artifact@v4
+        with:
+          name: source-distribution
+          path: dist
+      - name: Check release contents and tag
+        env:
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: |
+          python - <<'PY'
+          import email
+          import os
+          import tarfile
+          import zipfile
+          from pathlib import Path
+
+          version = os.environ["RELEASE_TAG"].removeprefix("v")
+          wheels = sorted(Path("dist").glob("*.whl"))
+          sdists = sorted(Path("dist").glob("*.tar.gz"))
+          assert len(wheels) == 4 and len(sdists) == 1, "Expected four wheels and one sdist"
+          assert {p.name.split("-")[2] for p in wheels} == {"cp310", "cp311", "cp312", "cp313"}
+          for path in wheels + sdists:
+              if path.suffix == ".whl":
+                  with zipfile.ZipFile(path) as archive:
+                      data = archive.read(next(p for p in archive.namelist() if p.endswith(".dist-info/METADATA")))
+              else:
+                  with tarfile.open(path) as archive:
+                      data = archive.extractfile(f"straw_queue-{version}/PKG-INFO").read()
+              metadata = email.message_from_bytes(data)
+              assert metadata["Name"] == "straw-queue", path
+              assert metadata["Version"] == version, f"{path}: version does not match {version}"
+              print(f"Verified {path.name}")
+          PY
+      - name: Publish distributions
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          print-hash: true

straw_queue-0.1.0/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+[中文版](CHANGELOG_zh.md)
+
+## 0.1.0
+
+Initial release of straw, a filesystem-based durable queue and shared tensor
+store for AI applications, distributed as `straw-queue` under the MIT license.
+
+- Rust storage, WAL, queue and ownership protocols with Python APIs. Multiple
+  machines can read and write through a shared filesystem; the initial shared
+  storage target is JuiceFS.
+- Packed immutable records and embedded manifests, batched publication and
+  size-based file rotation, avoiding a file per sample or tensor.
+- Bytes and custom codecs, NumPy/PyTorch tensors, checked lazy row reads and
+  native read sessions that reuse authenticated indices.
+- Shared tensor references across queues, tensor-level copy-on-write, explicit
+  reader/checkpoint ownership and optional online GC of unreferenced sealed packs.
+- Durable task leases, continuation progress, completion receipts and consumer
+  state, with WAL recovery and retained-checkpoint restore.
+- Local and multi-host I/O benchmarks, workload profiling and replay, an
+  independent protocol model, and process-crash/recovery checks.
+- Linux x86_64 wheels for CPython 3.10–3.13, automated artifact tests and PyPI
+  Trusted Publishing, standalone examples and paired English/Chinese guides.
+
+Each queue requires one externally fenced coordinator. Automatic failover,
+live-pack compaction, journal compaction and a general distributed application
+checkpoint transaction are not provided. Additional network filesystems require
+deployment validation. See the [filesystem requirements](docs/FILESYSTEM.md) and
+[protocol guarantees](docs/PROTOCOL_AND_VERIFICATION.md).

straw_queue-0.1.0/CHANGELOG_zh.md

@@ -0,0 +1,17 @@
+# 更新日志
+
+[English](CHANGELOG.md)
+
+## 0.1.0
+
+straw 首次发布：面向 AI 应用、基于文件系统的持久化队列与共享张量存储，以 `straw-queue` 包分发，采用 MIT 许可证。
+
+- Rust 实现存储、WAL、队列及所有权协议，提供 Python API。多台机器通过共享文件系统并行读写，首个共享存储目标为 JuiceFS。
+- 不可变记录与 manifest 打包存储，支持批量发布和按大小轮换文件，避免每个 sample 或张量单独创建文件。
+- 支持 bytes、自定义 codec、NumPy/PyTorch 张量、带校验的惰性按行读取，以及复用已认证索引的 native 读取会话。
+- 跨队列共享张量引用、张量级写时复制、显式 reader/checkpoint 所有权，以及可选在线 GC，回收无引用的封存 pack。
+- 持久化任务 lease、continuation 进度、完成 receipt 和消费者状态，支持 WAL 恢复与已保留 checkpoint 的恢复。
+- 本地与多机 I/O benchmark、应用负载采集与重放、独立协议模型，以及进程崩溃与恢复检查。
+- CPython 3.10–3.13 的 Linux x86_64 wheel、自动化产物测试与 PyPI Trusted Publishing、独立使用示例和配套中英文指南。
+
+每个队列需要一个由外部机制隔离保护的协调器。尚不提供自动故障切换、存活 pack 整理、日志压缩或通用分布式应用 checkpoint 事务。其他网络文件系统需要单独验证部署语义。详见[文件系统要求](docs/FILESYSTEM_zh.md)与[协议保证](docs/PROTOCOL_AND_VERIFICATION_zh.md)。

straw_queue-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,42 @@
+# Contributing
+
+[中文版](CONTRIBUTING_zh.md)
+
+Use Rust 1.89+ and CPython 3.10+. Install the PyTorch build appropriate to your
+environment, then install the development package:
+
+```sh
+python -m pip install -e '.[test,build,dev]'
+python -m black --check src tests examples tools verification
+python -m isort --check-only src tests examples tools verification
+cargo fmt --check
+cargo clippy --all-targets --features python -- -D warnings
+cargo test --locked
+python -m pytest
+python verification/ownership_model.py
+python verification/crash_campaign.py --output /tmp/straw-crashes.json
+```
+
+See [building](docs/BUILDING.md) for binary wheels and
+[verification](docs/VERIFICATION.md) for the scope of each check.
+Keep machine addresses, credentials, training scripts, payloads and raw
+experiment logs outside the project. Use a private temporary root for every
+filesystem test, with bounded data and cleanup after owned processes stop.
+
+Protocol changes must explain durable ordering, retry behavior, compatibility,
+reader ownership and failure handling. Add a reproducer for a real failure
+boundary, not just an assertion matching the implementation. Update the format
+and independent model where their abstractions change. Unknown versions must
+fail explicitly. Framing constants are part of the storage format; changing
+them requires an explicit format version.
+
+Custom application codecs and scheduling belong outside the Rust core. Prefer
+clear references and explicit lifetime signals to application-specific imports.
+Performance changes need comparable payload sizes, publication granularity,
+reader fanout, filesystem/cache conditions and actual measured byte counts.
+
+The project uses the [MIT license](LICENSE).
+
+User and contributor guides have matching English `.md` and Chinese `_zh.md`
+versions. Update both when behavior or examples change, preserve reciprocal
+language links, and keep API names, commands and protocol constants identical.

straw_queue-0.1.0/CONTRIBUTING_zh.md

@@ -0,0 +1,27 @@
+# 贡献指南
+
+[English](CONTRIBUTING.md)
+
+使用 Rust 1.89+、CPython 3.10+。先安装适合当前环境的 PyTorch，再安装开发包：
+
+```sh
+python -m pip install -e '.[test,build,dev]'
+python -m black --check src tests examples tools verification
+python -m isort --check-only src tests examples tools verification
+cargo fmt --check
+cargo clippy --all-targets --features python -- -D warnings
+cargo test --locked
+python -m pytest
+python verification/ownership_model.py
+python verification/crash_campaign.py --output /tmp/straw-crashes.json
+```
+
+二进制 wheel 见[构建文档](docs/BUILDING_zh.md)，每类检查的边界见[验证文档](docs/VERIFICATION_zh.md)。机器地址、凭据、训练脚本、载荷和原始实验日志应保存在项目外。每次文件系统测试使用独立私有临时目录，限制数据规模，并在自有进程停止后清理。
+
+协议修改必须说明持久化顺序、重试行为、兼容性、reader 所有权和失败处理。对真实故障边界添加复现用例，而非仅断言实现自身的行为。格式和独立模型的抽象发生变化时，同步更新它们。未知版本必须明确报错。Framing 常量属于存储格式，修改时需要显式变更格式版本。
+
+应用自定义 codec 和调度逻辑应放在 Rust 核心之外。使用清晰的引用和显式生命周期信号，避免引入应用专用依赖。性能修改需要可比的载荷大小、发布粒度、reader fanout、文件系统/缓存条件，以及实际测量的字节数。
+
+项目使用 [MIT 许可证](LICENSE)。
+
+用户与贡献者文档使用配套的英文 `.md` 和中文 `_zh.md`。行为或示例发生变化时同步更新两种语言，保留双向语言切换链接，并确保 API 名称、命令和协议常量一致。