straw-queue 0.1.2__tar.gz → 0.1.4__tar.gz

straw_queue-0.1.4/.github/workflows/tests.yml

@@ -0,0 +1,159 @@
+name: tests
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  formal:
+    name: TLA+ / shard ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+      - name: Check specifications and expected counterexamples
+        run: >-
+          python verification/runners/check_tla.py
+          --shard-index ${{ matrix.shard }} --shard-count 16
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: tla-verification-${{ matrix.shard }}
+          path: |
+            target/tla/report.json
+            target/tla/environment_contracts.json
+            target/tla/**/*.log
+            target/tla/**/*.tla
+            target/tla/**/*.cfg
+          if-no-files-found: error
+
+  verus:
+    name: Verus / shard ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.99.1
+        with:
+          components: rustc-dev, llvm-tools
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Verify pinned Rust contracts and semantic mutants
+        run: >-
+          python verification/runners/check_verus.py --download --timeout 600
+          --shard-index ${{ matrix.shard }} --shard-count 16
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: verus-verification-${{ matrix.shard }}
+          path: |
+            target/verus/report.json
+            target/verus/environment_contracts.json
+            target/verus/**/*.log
+            target/verus/**/*.rs
+          if-no-files-found: error
+
+  checks:
+    name: Rust tests and code checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install formatters
+        run: python -m pip install 'black>=24.8' 'isort>=5.13'
+      - name: Check environment contract inventory
+        run: python verification/runners/environment_manifest.py --check
+      - name: Check formatting
+        run: |
+          cargo fmt --check
+          python -m black --check src tests examples tools verification
+          python -m isort --check-only src tests examples tools verification
+      - name: Lint native core and Python bindings
+        run: |
+          cargo clippy --locked --all-targets -- -D warnings
+          cargo clippy --locked --all-targets --features python -- -D warnings
+      - name: Run Rust tests
+        run: cargo test --locked
+
+  python:
+    name: Full suite / Python ${{ matrix.python }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install build and test dependencies
+        run: |
+          python -m pip install 'build>=1.2' 'twine>=5' 'numpy>=1.24' 'pytest>=8'
+          python -m pip install 'torch>=2.3' --index-url https://download.pytorch.org/whl/cpu
+      - name: Build the wheel
+        run: python tools/release_build.py python -m build --wheel
+      - name: Check package metadata
+        run: python -m twine check dist/*
+      - name: Run all Python and protocol tests against the installed wheel
+        # Includes examples, multiprocess recovery, the ownership model, crash
+        # injection, a bounded I/O/GC benchmark and trace verification.
+        run: python tools/check_wheel.py dist/*.whl --tests --report wheel-check.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-report-py${{ matrix.python }}
+          path: wheel-check.json
+          if-no-files-found: warn
+
+  verification:
+    name: Verification passed
+    needs: [formal, verus, checks, python]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require every verification job to succeed
+        env:
+          RESULTS: ${{ toJSON(needs) }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+
+          results = json.loads(os.environ["RESULTS"])
+          assert set(results) == {"formal", "verus", "checks", "python"}
+          failed = {name: job["result"] for name, job in results.items() if job["result"] != "success"}
+          if failed:
+              raise SystemExit(f"Verification did not pass: {failed}")
+          PY

{straw_queue-0.1.2 → straw_queue-0.1.4}/.github/workflows/wheels.yml

@@ -7,7 +7,11 @@ on:
 permissions:
   contents: read
 jobs:
+  verification:
+    name: Verify the release source
+    uses: ./.github/workflows/tests.yml
   build:
+    needs: verification
     runs-on: ubuntu-latest
     timeout-minutes: 30
     strategy:
@@ -21,7 +25,11 @@ jobs:
           target: x86_64
           # The action adds --manylinux; do not also pass --compatibility in args.
           manylinux: '2_28'
+          before-script-linux: |
+            export CARGO_ENCODED_RUSTFLAGS="$(python3 tools/release_build.py --print-flags)"
           args: --release --locked --strip --out dist --interpreter /opt/python/cp${{ matrix.python-tag }}-cp${{ matrix.python-tag }}/bin/python
+      - name: Check release artifact privacy
+        run: python3 tools/check_artifact_privacy.py dist/*.whl
       - uses: actions/upload-artifact@v4
         with:
           name: wheel-cp${{ matrix.python-tag }}-linux-x86_64
@@ -63,6 +71,7 @@ jobs:
           name: wheel-check-cp${{ matrix.tag }}
           path: wheel-check.json
   sdist:
+    needs: verification
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -73,9 +82,10 @@ jobs:
           python-version: '3.12'
       - run: python -m pip install 'build>=1.2' 'twine>=5' 'numpy>=1.24' 'pytest>=8'
       - run: python -m pip install 'torch>=2.3' --index-url https://download.pytorch.org/whl/cpu
-      - run: python -m build --sdist
+      - run: python tools/release_build.py python -m build --sdist
       - run: python -m twine check dist/*
-      - run: python -m pip wheel --no-deps dist/*.tar.gz --wheel-dir rebuilt
+      - run: python tools/release_build.py python -m pip wheel --no-deps dist/*.tar.gz --wheel-dir rebuilt
+      - run: python tools/check_artifact_privacy.py dist/*.tar.gz rebuilt/*.whl
       - run: python -m twine check rebuilt/*
       - run: python tools/check_wheel.py rebuilt/*.whl --tests --report wheel-check.json
       - uses: actions/upload-artifact@v4
@@ -90,7 +100,7 @@ jobs:
           path: wheel-check.json
   publish:
     name: Publish to PyPI
-    needs: [smoke, sdist]
+    needs: [verification, smoke, sdist]
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     timeout-minutes: 10

straw_queue-0.1.4/CHANGELOG.md

@@ -0,0 +1,122 @@
+# Changelog
+
+[中文版](CHANGELOG_zh.md)
+
+## 0.1.4
+
+### Recovery and continuation
+
+- Preserve producer continuation payloads and independent reader, consumer and checkpoint roots during online GC. Sampling can resume from its durable continuation after process failure. Training applications must select and load a checkpoint with matching model/optimizer state and data cursors, and retain data needed for replay; straw does not automatically load a model checkpoint or recreate reclaimed data.
+- Resolve durable receipts before reopening old references when retrying accepted consumer-state, batch-plan, batch-ready and checkpoint-registration requests. Identical retries can succeed after those references have been legitimately reclaimed, without creating new ownership.
+- Persist staging cleanup intent, dependency recipes and generation-specific completion receipts. Recovery can finish an accepted handoff after a lost reply without releasing a later staging generation of the same reference.
+- Quarantine uncertain WAL writes, partial state application and caught Rust callback panics until recovery. Reject malformed journal/catalog events before appending, preserve confirmed prefixes during rebuild and reject changed pool/run or log identity. Harden counter arithmetic and deduplicate repeated task IDs before lease selection.
+
+### Storage and reads
+
+- Retry independent visibility delays in the header, footer, index, envelope, payload and tensor checksum chunks within one bounded budget. Every returned record remains authenticated; persistent corruption fails after the budget expires.
+- Reuse the authenticated index descriptor within a read attempt and buffer catalog replay with at most 64 KiB of read-ahead. Failed read attempts reopen their files. Replace recursive dependency walks with deduplicated worklists to support deep graphs.
+- Validate tensor dtype, shape and length before reading payloads. New tensors use 4 MiB checksum chunks; older tensors retain their stored chunk sizes. Checkpoint reactivation authenticates all reachable payloads outside the catalog lock with at most 1 MiB of payload scratch, then rechecks ownership and tombstones under the lock.
+- Preserve exact finite JSON floating-point values through serialization and recovery. Reject native payload providers whose traversals have the wrong length or swallow callback failures.
+
+### Garbage collection and publication safety
+
+- Retry directory barriers after unlink, including already absent files and missing parents. Batch shared-parent barriers and persist reclamation receipts only after deletion is durable. Preserve permanent tombstones; a cold-cache retain of a reclaimed root or dependency now returns `InvalidReference` before trying to read its missing payload.
+- Protect canonical paths and aliases, peer queues, active readers, checkpoints, producer continuations and pending publications. Public owner APIs reject internal `queue:` and `publication:` names. Offline cleanup preserves control files, rejects symlink cycles on Python 3.13 as well as earlier versions, and fails closed on uncertain shared-pool discovery; dangling orphan links remain eligible for cleanup.
+- Preserve dependency holds and retry identities through failed publication setup, synchronization, staging and sealing. Repair missing ancestor barriers, roll back unpublished tails before writer reuse and seal completed standalone reservations so they can be collected. A failed GC hook with partial WAL effects forces fresh recovery before further writes.
+
+### Writer retirement
+
+- Add explicit `retire_writer(..., policy="recover" | "abandon")` with a durable fence, operation registry and reusable recovery receipts. Recovery stages complete results before releasing preparation ownership; abandonment preserves existing queue, reader and staging roots. A writer holding its gate reports `WriterBusy`, and a fenced writer cannot resume writing. Retirement depends on the documented lock, fence-visibility and in-flight-I/O contracts; timeouts alone do not establish death.
+- Reject busy writer metrics, seal and close operations without waiting on an in-flight callback. Unknown legacy preparation owners remain protected and are reported for separate recovery.
+
+### Compatibility and migration
+
+- Packed-record and WAL byte framing remain unchanged, and legacy catalog entries still replay. New queue staging-handoff events require updated coordinators: do not downgrade a queue writer after those events are stored. Upgrade all GC participants before relying on canonical-alias protection; an old collector can still delete an aliased owned file. Mixed versions do not gain a uniform cleanup/retry guarantee; historical leaks and already reclaimed legacy manifests are not automatically repaired. See [format compatibility](docs/FORMAT.md#queue-staging-handoff-receipts).
+- Rust callers must use read-only coordinator accessors (`state()`, `journal()`, `store()`, `limits()`, `epoch()`), publish through a separate `Store`, and replace explicitly typed `&mut Catalog` callbacks with `&mut LockedCatalog<'_>`. `Config::gc_enabled` returns `Result<bool>`. Existing Python call signatures are retained, with the retirement APIs added.
+- Previously rounded floats cannot be reconstructed. Old rounded request hashes and retries generated under a different tensor chunk policy can conflict after upgrade; preserve retry identities and review the [stored-format rules](docs/FORMAT.md). Catalog recipes and receipts currently grow without compaction.
+
+### Known performance regressions
+
+- In the same JuiceFS workload with online GC, median write throughput fell 20.7% and acceptance time for 64 tasks rose 34.8% versus 0.1.3; reads and synchronous WAL appends showed no comparable decline. This short comparison used two rounds per build with warm caches. It was not profiled, so the changes cannot be attributed to a particular safety protocol or proof-related refactoring. See the [complete samples and build hashes](verification/measurements/20261026-release-comparison.json).
+
+### CI and verification
+
+- Scan source distributions and wheel contents, including native libraries, for local paths and recognizable credentials. Remap Rust build paths before compilation so released binaries omit local source, home and toolchain locations; CI checks artifacts before upload and rejects findings without printing matched secrets.
+- Require TLA+/Verus checks, semantic negative controls, environment-contract audits, Rust/Python tests, formatting and Clippy before release builds and publishing. TLC records its jar hash without enforcing a checksum; Verus logs distinguish successful proofs from expected semantic rejections. Verification files are organized by protocol, WAL, GC, storage and offline recovery, with independently reviewable environment assumptions.
+- Freeze new formal-verification work for this release. Integrated proofs cover specific production bodies and models; complete retain/reactivation, cross-log recovery, authentication and liveness composition remain deferred and receive regression/fault-injection coverage. See the [release scope and boundaries](docs/FORMAL_VERIFICATION.md#verification-scope-for-this-release). Passing tests does not complete these proofs, and prior performance measurements do not establish this build's throughput or latency.
+
+## 0.1.3
+
+- Retry temporarily zero-filled bytes anywhere in a published extent's header,
+  including partial and noncontiguous visibility gaps. Previously, only an
+  entirely zero-filled header triggered this retry. The retry limit and checksum
+  validation remain unchanged; other incorrect header bytes fail immediately.
+
+Existing callers and stored data remain compatible. Packed-record and WAL
+formats are unchanged.
+
+## 0.1.2
+
+- Stream large record payloads through bounded native write buffers. Records and
+  publications no longer need to fit the writer's scratch buffer; existing
+  packed-record framing and tensor reads are unchanged.
+- Remove fixed logical quotas on record and dependency counts, result sizes,
+  queue work, control messages, and metadata. Legacy limit arguments remain
+  accepted for API and queue-identity compatibility; `max_buffer_bytes` still
+  bounds temporary payload copies.
+- Retry transient shared-filesystem visibility gaps when a published extent's
+  length is visible before its header bytes. The reader reopens the file after
+  short delays; persistent failures, nonzero bad headers, and checksum errors
+  remain errors.
+
+Existing stored data and callers remain compatible. These write and read
+behaviors require 0.1.2; packed-record and WAL formats are unchanged.
+
+## 0.1.1
+
+- Persistent task scheduling with `TaskSpec.priority` (higher first),
+  `scheduling_key` (lower first), then FIFO. Both keys default to zero and accept
+  signed 64-bit integers. The Rust pending index selects tasks without reading
+  their payloads and reconstructs scheduling order during WAL recovery.
+- `Coordinator.yield_tasks()` atomically saves a batch of continuation inputs
+  and optional scheduling fields, ends their leases, and returns the tasks to
+  the pending queue without spending failure retries. Returned tasks join the
+  tail of their scheduling key's FIFO order. Identical requests are idempotent;
+  a stale lease rejects the entire batch.
+- `Coordinator.pending_tasks()` exposes pending task specifications in
+  acquisition order, with optional task-prefix and minimum-priority filters.
+  Applications coordinate paused snapshots and retain the referenced inputs.
+- Regression coverage for priority/FIFO recovery, stale leases, invalid
+  scheduling keys, and all-or-nothing batch recovery across WAL crash windows.
+
+Existing callers may omit the new scheduling fields. Packed-record and WAL
+framing are unchanged; the new APIs and scheduling behavior require 0.1.1.
+Pending-task snapshots alone do not provide a joint application checkpoint or
+arbitrary checkpoint rollback.
+
+## 0.1.0
+
+Initial release of straw, a filesystem-based durable queue and shared tensor
+store for AI applications, distributed as `straw-queue` under the MIT license.
+
+- Rust storage, WAL, queue and ownership protocols with Python APIs. Multiple
+  machines can read and write through a shared filesystem; the initial shared
+  storage target is JuiceFS.
+- Packed immutable records and embedded manifests, batched publication and
+  size-based file rotation, avoiding a file per sample or tensor.
+- Bytes and custom codecs, NumPy/PyTorch tensors, checked lazy row reads and
+  native read sessions that reuse authenticated indices.
+- Shared tensor references across queues, tensor-level copy-on-write, explicit
+  reader/checkpoint ownership and optional online GC of unreferenced sealed packs.
+- Durable task leases, continuation progress, completion receipts and consumer
+  state, with WAL recovery and retained-checkpoint restore.
+- Local and multi-host I/O benchmarks, workload profiling and replay, an
+  independent protocol model, and process-crash/recovery checks.
+- Linux x86_64 wheels for CPython 3.10–3.13, automated artifact tests and PyPI
+  Trusted Publishing, standalone examples and paired English/Chinese guides.
+
+Each queue requires one externally fenced coordinator. Automatic failover,
+live-pack compaction, journal compaction and a general distributed application
+checkpoint transaction are not provided. Additional network filesystems require
+deployment validation. See the [filesystem requirements](docs/FILESYSTEM.md) and
+[protocol guarantees](docs/PROTOCOL_AND_VERIFICATION.md).

straw_queue-0.1.4/CHANGELOG_zh.md

@@ -0,0 +1,83 @@
+# 更新日志
+
+[English](CHANGELOG.md)
+
+## 0.1.4
+
+### 恢复与续读
+
+- 在线 GC 保留 producer continuation，以及独立的 reader、consumer 和 checkpoint 根。采样进程失败后可以从已持久化的 continuation 恢复。训练应用仍须选择并加载 checkpoint，恢复相匹配的模型／optimizer 状态与数据游标，并保留重放需要的数据；straw 不会自动加载模型 checkpoint，也不能恢复已回收的数据。
+- 重试已接受的 consumer state、batch plan、batch ready 和 checkpoint registration 时，先查询持久回执，再决定是否读取旧引用。即使旧数据已合法回收，相同请求仍可返回原结果，不会重新建立所有权。
+- 持久化 staging 清理意图、依赖关系和对应代际的完成回执。丢失回复后，恢复可以完成已接受的交接，不会释放同一引用后来重新建立的 staging。
+- WAL 写入结果不确定、状态部分应用或捕获 Rust callback panic 后，保持隔离直至恢复。追加前拒绝畸形 journal／catalog 事件，重建时保护已确认前缀，并拒绝池／run／日志身份变化。补齐计数溢出检查，发放 lease 前对重复 task ID 去重。
+
+### 存储与读取
+
+- header、footer、index、envelope、payload 和 tensor checksum chunk 可在同一有限预算内重试各自的可见性延迟。返回的数据始终经过认证；持续损坏会在预算耗尽后报错。
+- 同次读取尝试复用认证索引的文件描述符，catalog replay 使用至多 64 KiB 预读；失败后重新打开文件。依赖遍历改用去重工作队列，支持深层依赖图。
+- 读取 payload 前检查 tensor dtype、shape 和长度。新 tensor 使用 4 MiB checksum chunk，旧数据沿用存储的 chunk 大小。checkpoint 重新激活在 catalog 锁外认证全部可达 payload，payload scratch 至多 1 MiB，随后在锁内重新检查所有权和 tombstone。
+- JSON 有限浮点数在序列化／恢复中精确往返。拒绝遍历长度不符或吞掉 callback 错误的原生 payload provider。
+
+### GC 与发布安全
+
+- unlink 后重试目录持久化屏障，包括文件已不存在或父目录缺失的情况。共享父目录的屏障合并执行，持久删除完成后才记录回收回执。永久保留 tombstone；冷缓存 retain 遇到已回收的根或依赖时，在读取缺失 payload 前返回 `InvalidReference`。
+- 保护规范路径及别名、其他队列、活跃 reader、checkpoint、producer continuation 和未完成 publication。公共 owner API 拒绝内部 `queue:`、`publication:` 名字。离线清理保留控制文件，在 Python 3.13 及更早版本中均拒绝符号链接环，共享池发现结果不确定时拒绝继续删除；无引用的悬空链接仍可清理。
+- 发布准备、同步、staging、sealing 失败时保留依赖保护和重试身份。补齐缺失的祖先目录屏障，writer 复用前回滚未发布尾部，并封存已完成的 standalone reservation 以便 GC。GC hook 报错且已影响 WAL 时，后续写入必须先重新恢复。
+
+### Writer 退休
+
+- 新增显式 `retire_writer(..., policy="recover" | "abandon")`，通过持久 fence、操作登记和可复用恢复回执处理旧 writer。recover 先保护完整结果再释放准备所有权；abandon 保留已有 queue、reader 和 staging 根。仍持有 gate 的 writer 返回 `WriterBusy`，已被 fence 的 writer 无法恢复写入。退休依赖已列明的锁、fence 可见性和在途 I/O 契约，不能仅凭超时判断进程死亡。
+- writer 正执行 callback 时，metrics、seal、close 立即返回 busy，不再等待。无法确认来源的历史 preparation owner 继续受保护，并报告为独立恢复事项。
+
+### 兼容与迁移
+
+- packed record 和 WAL 字节 framing 不变，旧 catalog 记录仍可重放。新增 queue staging handoff 事件需要新版 coordinator：一旦写入这些事件，不要降级 queue writer。依赖规范路径／别名保护前，应升级全部 GC 参与者；旧 collector 仍可能删除通过别名持有的文件。混用版本不具备统一的清理／重试保证；历史泄漏和已回收的旧 manifest 不会自动修复。见[格式兼容说明](docs/FORMAT_zh.md#队列-staging-交接凭据)。
+- Rust 调用者需改用 coordinator 只读 accessor（`state()`、`journal()`、`store()`、`limits()`、`epoch()`），通过独立 `Store` 发布，并将显式 `&mut Catalog` callback 改为 `&mut LockedCatalog<'_>`。`Config::gc_enabled` 返回 `Result<bool>`。现有 Python 调用签名保留，另增加退休 API。
+- 旧版本已舍入的浮点数无法还原；旧请求 hash 和使用不同 tensor chunk 策略生成的重试可能在升级后冲突。保留重试身份，并检查[存储格式规则](docs/FORMAT_zh.md)。catalog 依赖关系和回执暂未实现压缩，元数据会持续增长。
+
+### 已知性能回退
+
+- 相同 JuiceFS 工作负载、启用在线 GC 时，相比 0.1.3，写吞吐中位数下降 20.7%，接受 64 个任务的耗时增加 34.8%；读取和同步 WAL 追加未见同量级下降。此次简短对照每个构建只有两轮，缓存未清空；尚未做 profile，不能归因于某项安全协议或验证相关重构。见[完整样本与构建哈希](verification/measurements/20261026-release-comparison.json)。
+
+### CI 与验证边界
+
+- 扫描源码分发包与 wheel 内容（包括原生库），检查本机路径和可识别的凭据。编译时 remap Rust 路径，避免发布二进制携带本机源码、用户目录和工具链位置；CI 上传产物前检查，发现问题即拒绝，报告不输出命中的秘密内容。
+- release 构建和发布以前置检查为条件，包括 TLA+／Verus、语义负例、environment contract 审计、Rust／Python 测试、格式和 Clippy。TLC 只记录 jar 哈希、不做校验；Verus 日志明确区分证明通过与预期语义拒绝。验证目录按协议、WAL、GC、存储与离线恢复组织，环境假设可独立审查。
+- 本次发布冻结新增形式化验证。已集成证明覆盖具体生产函数和模型；完整 retain／reactivation、跨日志恢复、认证与活性组合延期，以回归和故障注入测试覆盖。见[本次发布范围与边界](docs/FORMAL_VERIFICATION_zh.md#本次发布的验证范围)。测试通过不等于这些证明已完成，历史性能测量也不能代表当前构建的吞吐或延迟。
+
+## 0.1.3
+
+- 已发布 extent 的文件头在任意位置出现暂时零值时都会触发重试，支持部分字节以及不连续位置尚不可见的情况。此前只有整个文件头为零时才会触发这类重试。重试上限和 checksum 校验保持不变；其他错误的文件头字节仍会立即报错。
+
+已有调用方和存储数据保持兼容，packed record 和 WAL 格式不变。
+
+## 0.1.2
+
+- 大记录的 payload 通过有界 native 写入缓冲区分块写入。记录和批量发布不再需要适配 writer 的临时缓冲区；原有 packed record 格式和张量读取方式不变。
+- 移除记录数、依赖数、结果大小、队列任务、控制消息及元数据的固定逻辑配额。旧的限制参数继续保留，以兼容 API 和队列身份；`max_buffer_bytes` 仍约束临时 payload 复制。
+- 共享文件系统上，已发布 extent 的文件长度可能先于文件头内容可见。reader 遇到这种短暂情况会延迟并重新打开文件；持续失败、非零坏文件头和 checksum 错误仍会报错。
+
+已有存储数据和调用方保持兼容。上述写入和读取行为需要 0.1.2；packed record 和 WAL 格式不变。
+
+## 0.1.1
+
+- 持久化任务调度：按 `TaskSpec.priority` 从高到低、`scheduling_key` 从低到高、再按 FIFO 顺序领取任务。两个字段默认均为 0，接受有符号 64 位整数。Rust pending 索引无需读取载荷即可选择任务，WAL 恢复时重建调度顺序。
+- 新增 `Coordinator.yield_tasks()`：原子保存一批 continuation 输入及可选调度字段，结束这些任务的 lease 并归还到 pending 队列，不消耗失败重试次数。归还的任务进入相同调度键的 FIFO 队尾；相同请求可幂等重试，任一 lease 失效则拒绝整批操作。
+- 新增 `Coordinator.pending_tasks()`：按领取顺序返回 pending 任务的 spec，支持按任务 ID 前缀及最低优先级过滤。应用负责协调暂停后的快照，并持有所引用的输入。
+- 补充优先级/FIFO 恢复、失效 lease、非法调度键，以及 WAL 崩溃窗口下批量操作全有或全无恢复的回归测试。
+
+已有调用方可以不指定新增调度字段。Packed record 与 WAL 的 framing 不变；新增 API 和调度行为需要 0.1.1。仅有 pending 任务快照尚不构成应用联合 checkpoint，也不提供任意 checkpoint 回退。
+
+## 0.1.0
+
+straw 首次发布：面向 AI 应用、基于文件系统的持久化队列与共享张量存储，以 `straw-queue` 包分发，采用 MIT 许可证。
+
+- Rust 实现存储、WAL、队列及所有权协议，提供 Python API。多台机器通过共享文件系统并行读写，首个共享存储目标为 JuiceFS。
+- 不可变记录与 manifest 打包存储，支持批量发布和按大小轮换文件，避免每个 sample 或张量单独创建文件。
+- 支持 bytes、自定义 codec、NumPy/PyTorch 张量、带校验的惰性按行读取，以及复用已认证索引的 native 读取会话。
+- 跨队列共享张量引用、张量级写时复制、显式 reader/checkpoint 所有权，以及可选在线 GC，回收无引用的封存 pack。
+- 持久化任务 lease、continuation 进度、完成 receipt 和消费者状态，支持 WAL 恢复与已保留 checkpoint 的恢复。
+- 本地与多机 I/O benchmark、应用负载采集与重放、独立协议模型，以及进程崩溃与恢复检查。
+- CPython 3.10–3.13 的 Linux x86_64 wheel、自动化产物测试与 PyPI Trusted Publishing、独立使用示例和配套中英文指南。
+
+每个队列需要一个由外部机制隔离保护的协调器。尚不提供自动故障切换、存活 pack 整理、日志压缩或通用分布式应用 checkpoint 事务。其他网络文件系统需要单独验证部署语义。详见[文件系统要求](docs/FILESYSTEM_zh.md)与[协议保证](docs/PROTOCOL_AND_VERIFICATION_zh.md)。

{straw_queue-0.1.2 → straw_queue-0.1.4}/CONTRIBUTING.md

@@ -13,8 +13,9 @@ cargo fmt --check
 cargo clippy --all-targets --features python -- -D warnings
 cargo test --locked
 python -m pytest
-python verification/ownership_model.py
-python verification/crash_campaign.py --output /tmp/straw-crashes.json
+python verification/runners/check_tla.py  # Java 17+
+python verification/protocol/checks/ownership_model.py
+python verification/wal/checks/crash_campaign.py --output /tmp/straw-crashes.json
 ```
 
 See [building](docs/BUILDING.md) for binary wheels and
@@ -40,3 +41,12 @@ The project uses the [MIT license](LICENSE).
 User and contributor guides have matching English `.md` and Chinese `_zh.md`
 versions. Update both when behavior or examples change, preserve reciprocal
 language links, and keep API names, commands and protocol constants identical.
+
+[TLA+ / Verus specifications and instructions](docs/FORMAL_VERIFICATION.md) describe model assumptions,
+code correspondence and required negative controls.
+
+When changing an environment assumption, update its stable ID in
+[`environment_contracts.json`](verification/environment_contracts.json), the
+[environment review](docs/ENVIRONMENT_MODELS.md), its consuming models and
+sensitivity controls. Keep algorithm policy, exploration bounds and measured
+percentiles separate from required environment guarantees.

{straw_queue-0.1.2 → straw_queue-0.1.4}/CONTRIBUTING_zh.md

@@ -12,8 +12,9 @@ cargo fmt --check
 cargo clippy --all-targets --features python -- -D warnings
 cargo test --locked
 python -m pytest
-python verification/ownership_model.py
-python verification/crash_campaign.py --output /tmp/straw-crashes.json
+python verification/runners/check_tla.py  # Java 17+
+python verification/protocol/checks/ownership_model.py
+python verification/wal/checks/crash_campaign.py --output /tmp/straw-crashes.json
 ```
 
 二进制 wheel 见[构建文档](docs/BUILDING_zh.md)，每类检查的边界见[验证文档](docs/VERIFICATION_zh.md)。机器地址、凭据、训练脚本、载荷和原始实验日志应保存在项目外。每次文件系统测试使用独立私有临时目录，限制数据规模，并在自有进程停止后清理。
@@ -25,3 +26,7 @@ python verification/crash_campaign.py --output /tmp/straw-crashes.json
 项目使用 [MIT 许可证](LICENSE)。
 
 用户与贡献者文档使用配套的英文 `.md` 和中文 `_zh.md`。行为或示例发生变化时同步更新两种语言，保留双向语言切换链接，并确保 API 名称、命令和协议常量一致。
+
+[TLA+ / Verus 规范与运行说明](docs/FORMAL_VERIFICATION_zh.md)包含模型假设、代码对应关系与必须失败的负向对照。
+
+修改环境假设时，同步更新 [`environment_contracts.json`](verification/environment_contracts.json) 中的稳定编号、[环境审查文档](docs/ENVIRONMENT_MODELS_zh.md)、消费该假设的模型和负向对照。区分算法策略、搜索边界、实测分位数与环境必需保证。

{straw_queue-0.1.2 → straw_queue-0.1.4}/Cargo.lock

@@ -370,7 +370,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "straw"
-version = "0.1.2"
+version = "0.1.4"
 dependencies = [
  "pyo3",
  "serde",

straw_queue-0.1.4/Cargo.toml

@@ -0,0 +1,65 @@
+[package]
+name = "straw"
+version = "0.1.4"
+edition = "2024"
+rust-version = "1.89"
+license = "MIT"
+repository = "https://github.com/zhuzilin/straw"
+description = "Durable shared-filesystem queues and immutable packed records"
+include = [
+    "Cargo.toml", "Cargo.lock", "pyproject.toml", "README*.md", "LICENSE",
+    "CHANGELOG*.md", "CONTRIBUTING*.md", ".github/workflows/*.yml",
+    "rust/*.rs", "src/**/*.py", "docs/**/*.md", "examples/*.py",
+    "tests/*.py", "tests/native.rs", "verification/**/*.md", "verification/**/*.py", "verification/**/*.rs", "verification/**/*.c", "verification/environment_contracts.json", "verification/measurements/*.json", "verification/**/tla/*.tla", "verification/**/tla/*.cfg", "tools/*.py",
+]
+readme = "README.md"
+
+[lib]
+name = "straw"
+path = "rust/lib.rs"
+crate-type = ["rlib", "cdylib"]
+
+[features]
+default = []
+python = ["dep:pyo3", "pyo3/extension-module"]
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = ["float_roundtrip"] }
+sha2 = "0.10"
+uuid = { version = "1.23", features = ["v4"] }
+pyo3 = { version = "0.23.5", optional = true }
+
+[dev-dependencies]
+tempfile = "3.27"
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ["cfg(straw_verus)"] }
+
+[[example]]
+name = "large_extent_probe"
+path = "verification/storage/probes/large_extent_probe.rs"
+
+[[example]]
+name = "reactivation_probe"
+path = "verification/protocol/probes/reactivation_probe.rs"
+
+[[example]]
+name = "standalone_extent_probe"
+path = "verification/storage/probes/standalone_extent_probe.rs"
+
+[[example]]
+name = "handoff_extent_probe"
+path = "verification/protocol/probes/handoff_extent_probe.rs"
+
+[[example]]
+name = "workflow_extent_probe"
+path = "verification/protocol/probes/workflow_extent_probe.rs"
+
+[[example]]
+name = "queue_record_probe"
+path = "verification/protocol/probes/queue_record_probe.rs"
+
+[[example]]
+name = "positioned_extent_probe"
+path = "verification/storage/probes/positioned_extent_probe.rs"

{straw_queue-0.1.2 → straw_queue-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: straw-queue
-Version: 0.1.2
+Version: 0.1.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
@@ -176,6 +176,7 @@ workers stop. [Multi-host benchmarks and trace replay](docs/BENCHMARKS.md).
 - [Records, tensors, tasks, recovery and GC](docs/USAGE.md)
 - [Filesystem deployment and file-count planning](docs/FILESYSTEM.md)
 - [Protocol, prior work and correctness boundaries](docs/PROTOCOL_AND_VERIFICATION.md)
+- [TLA+ / Verus verification](docs/FORMAL_VERIFICATION.md): layered coverage, production Rust proofs and counterexamples
 - [Executable verification](docs/VERIFICATION.md) and [binary format](docs/FORMAT.md)
 - [Wheel builds and release checks](docs/BUILDING.md)
 - [Release notes](CHANGELOG.md)
@@ -189,7 +190,8 @@ workers stop. [Multi-host benchmarks and trace replay](docs/BENCHMARKS.md).
 - [Wheels](.github/workflows/wheels.yml): build and verify Linux x86_64 wheels
   for CPython 3.10–3.13 (manylinux/glibc 2.28+), plus a source archive and rebuild
   check. Download the outputs from the run's **Artifacts** section. Version-tag
-  pushes publish the verified artifacts to PyPI through Trusted Publishing.
+  pushes publish the verified artifacts to PyPI through Trusted Publishing. The
+  same-commit complete verification workflow must pass before package uploads.
 
 Both workflows support **Actions → select workflow → Run workflow**. Tests also
 run on pushes and pull requests; wheels run on pull requests and `v*` tags.

{straw_queue-0.1.2 → straw_queue-0.1.4}/README.md

@@ -149,6 +149,7 @@ workers stop. [Multi-host benchmarks and trace replay](docs/BENCHMARKS.md).
 - [Records, tensors, tasks, recovery and GC](docs/USAGE.md)
 - [Filesystem deployment and file-count planning](docs/FILESYSTEM.md)
 - [Protocol, prior work and correctness boundaries](docs/PROTOCOL_AND_VERIFICATION.md)
+- [TLA+ / Verus verification](docs/FORMAL_VERIFICATION.md): layered coverage, production Rust proofs and counterexamples
 - [Executable verification](docs/VERIFICATION.md) and [binary format](docs/FORMAT.md)
 - [Wheel builds and release checks](docs/BUILDING.md)
 - [Release notes](CHANGELOG.md)
@@ -162,7 +163,8 @@ workers stop. [Multi-host benchmarks and trace replay](docs/BENCHMARKS.md).
 - [Wheels](.github/workflows/wheels.yml): build and verify Linux x86_64 wheels
   for CPython 3.10–3.13 (manylinux/glibc 2.28+), plus a source archive and rebuild
   check. Download the outputs from the run's **Artifacts** section. Version-tag
-  pushes publish the verified artifacts to PyPI through Trusted Publishing.
+  pushes publish the verified artifacts to PyPI through Trusted Publishing. The
+  same-commit complete verification workflow must pass before package uploads.
 
 Both workflows support **Actions → select workflow → Run workflow**. Tests also
 run on pushes and pull requests; wheels run on pull requests and `v*` tags.

{straw_queue-0.1.2 → straw_queue-0.1.4}/README_zh.md

@@ -101,6 +101,7 @@ Benchmark 会校验读取结果，测量持久化发布与接收延迟，报告
 - [记录、张量、任务、恢复与 GC](docs/USAGE_zh.md)
 - [文件系统部署与文件数规划](docs/FILESYSTEM_zh.md)
 - [协议、参考工作与正确性边界](docs/PROTOCOL_AND_VERIFICATION_zh.md)
+- [TLA+ / Verus 形式化验证](docs/FORMAL_VERIFICATION_zh.md)：分层覆盖、实际 Rust 证明、反例与边界
 - [可执行验证](docs/VERIFICATION_zh.md)与[二进制格式](docs/FORMAT_zh.md)
 - [构建 wheel 与发布检查](docs/BUILDING_zh.md)
 - [更新日志](CHANGELOG_zh.md)
@@ -109,7 +110,7 @@ Benchmark 会校验读取结果，测量持久化发布与接收延迟，报告
 ## GitHub 构建与测试
 
 - [全套测试](.github/workflows/tests.yml)：Rust 测试、格式检查和 Clippy，以及 CPython 3.10–3.13 下针对已安装 wheel 的全部 Python 测试、所有权模型、故障注入、示例和有界 I/O/GC benchmark。
-- [Wheel 构建](.github/workflows/wheels.yml)：构建并验证 CPython 3.10–3.13 的 Linux x86_64 wheel（manylinux/glibc 2.28+），同时生成源码包并检查能否重新构建。从工作流运行页面的 **Artifacts** 下载产物。推送版本 tag 时，通过 Trusted Publishing 将已验证产物发布到 PyPI。
+- [Wheel 构建](.github/workflows/wheels.yml)：构建并验证 CPython 3.10–3.13 的 Linux x86_64 wheel（manylinux/glibc 2.28+），同时生成源码包并检查能否重新构建。从工作流运行页面的 **Artifacts** 下载产物。推送版本 tag 时，通过 Trusted Publishing 将已验证产物发布到 PyPI；同一提交的完整验证工作流必须先通过，才能上传发布产物。
 
 两条工作流都可以通过 **Actions → 选择工作流 → Run workflow** 手动触发。全套测试还会在 push 和 pull request 时运行；wheel 构建会在 pull request 和 `v*` tag 时运行。托管 CI 使用本地文件系统和 CPU PyTorch；多客户端 JuiceFS 部署验证与 GPU 训练使用[独立检查](docs/VERIFICATION_zh.md)。详见[构建与发布说明](docs/BUILDING_zh.md)。
 

{straw_queue-0.1.2 → straw_queue-0.1.4}/docs/APPLICATIONS.md

@@ -49,3 +49,5 @@ optimizer step after rollback is distinct from duplicate logical acceptance.
 Application integration tests should cover their real schemas, abort/resume,
 shared tensor lifetime, multi-reader completion and whole-job recovery. They
 complement the [core protocol checks](VERIFICATION.md); they do not replace them.
+
+Training recovery must use the data state from the selected committed model checkpoint, not automatically the latest queue consumer state. Post-checkpoint results need replay owners before processing acknowledgement allows GC; see [restart semantics](USAGE.md#what-resumes-after-a-restart).

{straw_queue-0.1.2 → straw_queue-0.1.4}/docs/APPLICATIONS_zh.md

@@ -26,3 +26,5 @@ straw 核心负责字节、引用、持久化任务、结果接收和存储生
 联合 checkpoint 应先保留 pending、buffered、prefetched 数据的全部依赖，确保模型/优化器/RNG 和队列视图组件持久化，再发布指明精确版本的最终已提交 manifest。新 checkpoint 持久化之前保留上一份完整 checkpoint。仅模型的 latest 标记或队列消费确认无法替代这一协议。回滚后重做尚未提交的优化器步骤，与重复接收同一个逻辑结果是不同问题。
 
 应用集成测试应覆盖真实 schema、中断/续跑、共享张量生命周期、多 reader 完成信号和整个任务恢复。这些测试补充[核心协议验证](VERIFICATION_zh.md)，不能替代它。
+
+续训必须使用所选已提交模型 checkpoint 对应的数据状态，不能直接套用队列最新 consumer state。Checkpoint 后的新结果应在 processed 确认允许 GC 前建立 replay owner；见[重启语义](USAGE_zh.md#重启后分别续上什么)。