straw-queue 0.1.2__tar.gz → 0.1.3__tar.gz

{straw_queue-0.1.2 → straw_queue-0.1.3}/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 [中文版](CHANGELOG_zh.md)
 
+## 0.1.3
+
+- Retry temporarily zero-filled bytes anywhere in a published extent's header,
+  including partial and noncontiguous visibility gaps. Previously, only an
+  entirely zero-filled header triggered this retry. The retry limit and checksum
+  validation remain unchanged; other incorrect header bytes fail immediately.
+
+Existing callers and stored data remain compatible. Packed-record and WAL
+formats are unchanged.
+
 ## 0.1.2
 
 - Stream large record payloads through bounded native write buffers. Records and

{straw_queue-0.1.2 → straw_queue-0.1.3}/CHANGELOG_zh.md

@@ -2,6 +2,12 @@
 
 [English](CHANGELOG.md)
 
+## 0.1.3
+
+- 已发布 extent 的文件头在任意位置出现暂时零值时都会触发重试，支持部分字节以及不连续位置尚不可见的情况。此前只有整个文件头为零时才会触发这类重试。重试上限和 checksum 校验保持不变；其他错误的文件头字节仍会立即报错。
+
+已有调用方和存储数据保持兼容，packed record 和 WAL 格式不变。
+
 ## 0.1.2
 
 - 大记录的 payload 通过有界 native 写入缓冲区分块写入。记录和批量发布不再需要适配 writer 的临时缓冲区；原有 packed record 格式和张量读取方式不变。

{straw_queue-0.1.2 → straw_queue-0.1.3}/Cargo.lock

@@ -370,7 +370,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "straw"
-version = "0.1.2"
+version = "0.1.3"
 dependencies = [
  "pyo3",
  "serde",

{straw_queue-0.1.2 → straw_queue-0.1.3}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "straw"
-version = "0.1.2"
+version = "0.1.3"
 edition = "2024"
 rust-version = "1.89"
 license = "MIT"

{straw_queue-0.1.2 → straw_queue-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: straw-queue
-Version: 0.1.2
+Version: 0.1.3
 Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3

{straw_queue-0.1.2 → straw_queue-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "straw-queue"
-version = "0.1.2"
+version = "0.1.3"
 description = "Rust durable queues for AI tensors and large immutable records"
 readme = "README.md"
 requires-python = ">=3.10"

{straw_queue-0.1.2 → straw_queue-0.1.3}/rust/store.rs

@@ -332,12 +332,18 @@ impl Config {
                 if header == HEADER {
                     if attempt > 0 {
                         eprintln!(
-                            "Straw extent visible after {attempt} retries; path={path:?}, offset={offset}"
+                            "straw extent visible after {attempt} retries; path={path:?}, offset={offset}"
                         );
                     }
                     return Ok(f);
                 }
-                let all_zero = header.iter().all(|b| *b == 0);
+                // A header spanning cached pages can be only partly visible:
+                // missing bytes read as zero while the other bytes already match.
+                // Reopen within the same bounded window, then validate normally.
+                let zero_filled = header
+                    .iter()
+                    .zip(HEADER)
+                    .all(|(observed, expected)| *observed == 0 || observed == expected);
                 (
                     Error::new(
                         "CorruptData",
@@ -347,7 +353,7 @@ impl Config {
                             hex(HEADER)
                         ),
                     ),
-                    all_zero,
+                    zero_filled,
                 )
             };
             // Close before sleeping so the next attempt gets a fresh open and
@@ -358,7 +364,7 @@ impl Config {
             }
             let delay = EXTENT_VISIBILITY_DELAYS_MS[attempt];
             eprintln!(
-                "Straw extent visibility retry {}/{} after {delay}ms: {error}",
+                "straw extent visibility retry {}/{} after {delay}ms: {error}",
                 attempt + 1,
                 EXTENT_VISIBILITY_DELAYS_MS.len()
             );

{straw_queue-0.1.2 → straw_queue-0.1.3}/tests/test_extent_visibility.py

@@ -11,13 +11,16 @@ from straw.errors import CorruptData
 
 
 @contextmanager
-def delayed_header(root, segment):
+def delayed_header(root, segment, zero_positions=range(12)):
     path = root / segment.path
     with path.open("r+b") as f:
         f.seek(segment.offset)
         header = f.read(12)
         f.seek(segment.offset)
-        f.write(bytes(12))
+        visible_header = bytearray(header)
+        for position in zero_positions:
+            visible_header[position] = 0
+        f.write(visible_header)
         f.flush()
         os.fsync(f.fileno())
 
@@ -37,7 +40,20 @@ def delayed_header(root, segment):
 
 
 @pytest.mark.parametrize("entry", ["inspect", "read_record", "read", "validate", "session_read", "dependency"])
-def test_published_zero_header_is_reopened_and_revalidated(tmp_path, capfd, entry):
+@pytest.mark.parametrize(
+    "zero_positions",
+    [
+        tuple(range(12)),
+        tuple(range(4)),
+        tuple(range(4, 12)),
+        (3, 4, 5),
+        (0, 3, 6, 8),
+        # The last three header bytes are already zero. Exercise each nonzero
+        # byte independently as well as contiguous and scattered gaps above.
+        *((position,) for position in range(9)),
+    ],
+)
+def test_published_zero_header_is_reopened_and_revalidated(tmp_path, capfd, entry, zero_positions):
     writer = SharedFilesystemStore(tmp_path, "visibility")
     writer.publish([Record("prefix", b"prefix")], submission_id="prefix")
     ref = writer.publish([Record("result", b"immutable payload")], submission_id="result")
@@ -46,7 +62,7 @@ def test_published_zero_header_is_reopened_and_revalidated(tmp_path, capfd, entr
     parent = writer.publish([Record("parent", b"parent")], dependencies=[ref], submission_id="parent")
     reader = SharedFilesystemStore(tmp_path, "visibility")
     assert ref.manifest.segment.offset > 0
-    with delayed_header(tmp_path, ref.manifest.segment):
+    with delayed_header(tmp_path, ref.manifest.segment, zero_positions):
         if entry == "inspect":
             reader.inspect_segment(ref.manifest.segment, verify_payload=True)
         elif entry == "read_record":
@@ -60,30 +76,51 @@ def test_published_zero_header_is_reopened_and_revalidated(tmp_path, capfd, entr
         else:
             with reader.read_session() as session:
                 assert list(session.read(ref))[0].payload == b"immutable payload"
-    assert "Straw extent visibility retry" in capfd.readouterr().err
+    assert "straw extent visibility retry" in capfd.readouterr().err
 
 
-def test_persistent_zero_header_still_fails_after_bounded_retries(tmp_path, capfd):
+@pytest.mark.parametrize("zero_positions", [tuple(range(12)), tuple(range(4)), (3, 4, 5), (0, 3, 6, 8)])
+def test_persistent_zero_header_still_fails_after_bounded_retries(tmp_path, capfd, zero_positions):
     store = SharedFilesystemStore(tmp_path, "persistent")
     ref = store.publish([Record("r", b"payload")], submission_id="r")
     segment = ref.manifest.segment
     with (tmp_path / segment.path).open("r+b") as f:
-        f.seek(segment.offset)
-        f.write(bytes(12))
+        for position in zero_positions:
+            f.seek(segment.offset + position)
+            f.write(b"\x00")
     with pytest.raises(CorruptData, match="Invalid segment header"):
         store.validate(ref)
     log = capfd.readouterr().err
-    assert log.count("Straw extent visibility retry") == 5
+    assert log.count("straw extent visibility retry") == 5
     assert "visible after" not in log
 
 
-def test_nonzero_corruption_is_not_treated_as_delayed_publication(tmp_path, capfd):
+@pytest.mark.parametrize("corrupt_position", range(12))
+def test_nonzero_corruption_is_not_treated_as_delayed_publication(tmp_path, capfd, corrupt_position):
     store = SharedFilesystemStore(tmp_path, "corrupt")
     ref = store.publish([Record("r", b"payload")], submission_id="r")
     segment = ref.manifest.segment
     with (tmp_path / segment.path).open("r+b") as f:
         f.seek(segment.offset)
-        f.write(b"BROKENHEADER")
+        f.write(bytes(12))
+        f.seek(segment.offset + corrupt_position)
+        f.write(b"\xff")
     with pytest.raises(CorruptData, match="Invalid segment header"):
         store.validate(ref)
     assert "visibility retry" not in capfd.readouterr().err
+
+
+def test_recovered_header_does_not_bypass_payload_validation(tmp_path, capfd):
+    store = SharedFilesystemStore(tmp_path, "corrupt-payload")
+    ref = store.publish([Record("r", b"immutable payload")], submission_id="r")
+    segment = ref.manifest.segment
+    path = tmp_path / segment.path
+    data = path.read_bytes()
+    payload_offset = data.index(b"immutable payload")
+    with path.open("r+b") as f:
+        f.seek(payload_offset)
+        f.write(b"X")
+    with delayed_header(tmp_path, segment, range(4)):
+        with pytest.raises(CorruptData, match="checksum mismatch"):
+            store.inspect_segment(segment, verify_payload=True)
+    assert "visibility retry" in capfd.readouterr().err