zerobucket 0.26.0__tar.gz → 0.26.1__tar.gz

{zerobucket-0.26.0 → zerobucket-0.26.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: zerobucket
-Version: 0.26.0
+Version: 0.26.1
 Summary: Database-native image storage. Your database. Your images. Zero buckets.
 Project-URL: Homepage, https://github.com/KedarGhadyalji/ZeroBucket
 Project-URL: Repository, https://github.com/KedarGhadyalji/ZeroBucket

{zerobucket-0.26.0 → zerobucket-0.26.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "zerobucket"
-version = "0.26.0"
+version = "0.26.1"
 description = "Database-native image storage. Your database. Your images. Zero buckets."
 readme = "README.md"
 license = "MIT"

{zerobucket-0.26.0 → zerobucket-0.26.1}/src/zerobucket/__init__.py

@@ -40,7 +40,7 @@ from .types import (
     ImageMetadata,
 )
 
-__version__ = "0.26.0"
+__version__ = "0.26.1"
 
 __all__ = [
     "ZeroBucket",

{zerobucket-0.26.0 → zerobucket-0.26.1}/src/zerobucket/adapters/sqlite.py

@@ -48,11 +48,14 @@ than left for someone to wonder about while reading unfamiliar code:
   a dynamically-sized `IN (?, ?, ...)` placeholder list instead.
 - `RETURNING` IS supported (SQLite >= 3.35, released March 2021) and is
   used the same way the Postgres adapter uses it.
-- get_stream() uses `sqlite3.Connection.blobopen()` (Python >= 3.11), a
-  genuine incremental-BLOB-I/O API -- not a workaround built on top of
-  substring()-style range queries the way the Postgres adapter's
-  version is. This is arguably a more natural fit for streaming than
-  the Postgres implementation, not a lesser one.
+- get_stream() uses `sqlite3.Connection.blobopen()` where it exists
+  (Python >= 3.11), a genuine incremental-BLOB-I/O API -- not a
+  workaround built on top of substring()-style range queries the way the
+  Postgres adapter's version is. This package supports Python 3.10 too,
+  which has no blobopen(); there get_stream() reads the same bytes with
+  substr() range queries inside ONE held read transaction, which gives
+  the same snapshot behaviour (see `_stream_with_substr`). Both paths are
+  covered by tests/test_sqlite_stream_fallback.py on every Python.
 """
 
 from __future__ import annotations
@@ -67,6 +70,13 @@ from ..exceptions import StorageError
 from ..object_storage import ObjectStorage
 from .base import StorageBackend, StoredRecord, StoredRecordMetadata
 
+# Connection.blobopen() (incremental BLOB I/O) was added in Python 3.11; this package
+# supports 3.10. Where it is missing, get_stream() reads the same bytes with substr()
+# range queries inside one held read transaction. A module-level flag, not an inline
+# hasattr(), so the fallback can be exercised on newer Pythons too (the tests switch
+# it off).
+_HAS_BLOBOPEN = hasattr(sqlite3.Connection, "blobopen")
+
 DEFAULT_STREAM_CHUNK_SIZE = 1024 * 1024
 
 # 10 seconds. See _connect()'s docstring for why this is set explicitly
@@ -250,6 +260,60 @@ def _now_iso() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def _stream_with_substr(
+    conn: sqlite3.Connection,
+    table: str,
+    rowid: int,
+    total_size: int,
+    chunk_size: int,
+    image_id: str,
+    owns_connection: bool,
+) -> Iterator[bytes]:
+    """get_stream() for Pythons without Connection.blobopen() (3.10).
+
+    Reads the bytes with `substr()` range queries, one per chunk, but inside ONE read
+    transaction held open for the whole stream. In WAL mode that transaction pins a
+    snapshot, so a stream that has started keeps returning the original bytes even if
+    another connection deletes the row meanwhile: the same behaviour as the blobopen()
+    path (and unlike the Postgres and async SQLite adapters, whose per-chunk queries
+    each see the current state). If the caller passed their own connection that is
+    already inside a transaction, that transaction is used as it is and left open.
+
+    `table` is one of two constants chosen by the caller, never user input.
+    """
+    began = False
+    try:
+        if not conn.in_transaction:
+            try:
+                conn.execute("BEGIN")
+                began = True
+            except Exception as exc:  # noqa: BLE001
+                raise StorageError(f"Failed to stream image: {exc}") from exc
+        delivered = 0
+        while delivered < total_size:
+            want = min(chunk_size, total_size - delivered)
+            try:
+                row = conn.execute(
+                    f"SELECT substr(data, ?, ?) FROM {table} WHERE rowid = ?;",
+                    (delivered + 1, want, rowid),  # substr() is 1-indexed
+                ).fetchone()
+            except Exception as exc:  # noqa: BLE001
+                raise StorageError(f"Failed to stream image: {exc}") from exc
+            chunk = bytes(row[0]) if row is not None and row[0] is not None else b""
+            if not chunk:
+                raise StorageError(
+                    f"Image {image_id!r} was deleted while streaming "
+                    f"(delivered {delivered} of {total_size} bytes)."
+                )
+            yield chunk
+            delivered += len(chunk)
+    finally:
+        if began:
+            conn.rollback()  # ends the read transaction; nothing was written
+        if owns_connection:
+            conn.close()
+
+
 class SQLiteBackend(StorageBackend):
     """Storage backend for SQLite. See module docstring for what's
     implemented in this phase and what isn't yet.
@@ -642,8 +706,10 @@ class SQLiteBackend(StorageBackend):
         chunk_size: int = DEFAULT_STREAM_CHUNK_SIZE,
         connection: sqlite3.Connection | None = None,
     ) -> Iterator[bytes] | None:
-        """Uses `sqlite3.Connection.blobopen()` -- a real incremental-
-        BLOB-read API, not a range-query workaround. Unlike the Postgres
+        """Uses `sqlite3.Connection.blobopen()` on Python 3.11+ -- a real
+        incremental-BLOB-read API, not a range-query workaround -- and
+        `substr()` range queries in one held read transaction on 3.10,
+        which has no blobopen() (see `_stream_with_substr`). Unlike the Postgres
         adapter (which does one `_run()` call PER CHUNK, each its own
         pooled connection), this holds ONE connection open for the
         WHOLE stream -- necessary because the Blob handle is tied to
@@ -663,7 +729,8 @@ class SQLiteBackend(StorageBackend):
         deleted by another connection mid-stream causes the NEXT chunk
         fetch to see no row and raise StorageError. On SQLite, this
         method holds ONE connection open for the whole stream, and
-        `blobopen()`'s underlying read transaction gives it a
+        the read transaction (blobopen()'s own on Python 3.11+, an explicit
+        one on 3.10, where blobopen() does not exist) gives it a
         consistent snapshot (in WAL mode) of the row as it was when
         streaming began -- a concurrent DELETE from another connection
         does NOT interrupt or truncate an in-progress stream here; the
@@ -732,6 +799,17 @@ class SQLiteBackend(StorageBackend):
             table = "zerobucket_images"
 
         def generator() -> Iterator[bytes]:
+            if not _HAS_BLOBOPEN:
+                yield from _stream_with_substr(
+                    conn,
+                    table,
+                    rowid,
+                    total_size,
+                    chunk_size,
+                    image_id,
+                    owns_connection,
+                )
+                return
             try:
                 blob = conn.blobopen(table, "data", rowid, readonly=True)
             except Exception as exc:  # noqa: BLE001

zerobucket-0.26.1/tests/test_sqlite_stream_fallback.py

@@ -0,0 +1,160 @@
+"""SQLiteBackend.get_stream() on Pythons without Connection.blobopen() (3.10).
+
+blobopen() (incremental BLOB I/O) arrived in Python 3.11, but this package supports 3.10,
+so get_stream() has a fallback that reads the same bytes with substr() range queries
+inside one held read transaction. These tests run every scenario through BOTH
+implementations by switching the module flag, so the fallback is exercised on 3.11 and
+3.12 as well, not only on the one Python where it is the only option. (On 3.10 the
+"blobopen" variant is skipped: there is nothing to switch to.)
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import sqlite3
+import tempfile
+
+import pytest
+from PIL import Image as PILImage
+
+from zerobucket import ZeroBucket
+from zerobucket.adapters import sqlite as sqlite_mod
+from zerobucket.adapters.sqlite import SQLiteBackend
+from zerobucket.exceptions import StorageError
+
+
+@pytest.fixture(params=["blobopen", "substr"])
+def impl(request, monkeypatch):
+    if request.param == "blobopen" and not hasattr(sqlite3.Connection, "blobopen"):
+        pytest.skip("this Python has no Connection.blobopen (3.10): only the fallback exists")
+    monkeypatch.setattr(sqlite_mod, "_HAS_BLOBOPEN", request.param == "blobopen")
+    return request.param
+
+
+@pytest.fixture
+def path():
+    p = tempfile.mktemp(suffix=".db")
+    yield p
+    for ext in ("", "-wal", "-shm"):
+        if os.path.exists(p + ext):
+            os.remove(p + ext)
+
+
+@pytest.fixture(params=["classic", "dedup"])
+def zb(request, path):
+    z = ZeroBucket(backend=SQLiteBackend(path, dedup=(request.param == "dedup")))
+    yield z
+    z.close()
+
+
+def _noise_jpeg(side: int = 300) -> bytes:
+    """A JPEG of random pixels: large for its dimensions and different on every call."""
+    img = PILImage.frombytes("RGB", (side, side), os.urandom(side * side * 3))
+    buf = io.BytesIO()
+    img.save(buf, format="JPEG", quality=95)
+    return buf.getvalue()
+
+
+def _delete_the_bytes_from_another_connection(path: str, image_id: str, dedup: bool) -> None:
+    """Delete the row get_stream() actually reads from: the image row, or in dedup mode the
+    shared blob row (deleting only the reference would leave the bytes in place and prove
+    nothing)."""
+    other = sqlite3.connect(path)
+    try:
+        if dedup:
+            # these tests store exactly one distinct image
+            other.execute("DELETE FROM zerobucket_blobs")
+        else:
+            other.execute("DELETE FROM zerobucket_images WHERE id = ?", (image_id,))
+        other.commit()
+    finally:
+        other.close()
+
+
+def test_the_flag_reflects_this_python():
+    assert sqlite_mod._HAS_BLOBOPEN == hasattr(sqlite3.Connection, "blobopen")
+
+
+def test_streams_the_exact_bytes_at_awkward_chunk_sizes(impl, zb):
+    data = _noise_jpeg()
+    image_id = zb.put(data)
+    for chunk_size in (1_000, 97, 4_096, len(data), len(data) + 5, 1_048_576):
+        chunks = list(zb.get_stream(image_id, chunk_size=chunk_size))
+        assert b"".join(chunks) == data, chunk_size
+        # every chunk but the last is exactly chunk_size, and none is empty
+        assert all(len(c) == chunk_size for c in chunks[:-1]), chunk_size
+        assert 0 < len(chunks[-1]) <= chunk_size, chunk_size
+
+
+def test_a_stream_in_progress_keeps_its_snapshot_when_the_row_is_deleted_elsewhere(impl, zb, path):
+    data = _noise_jpeg()
+    image_id = zb.put(data)
+    stream = zb.get_stream(image_id, chunk_size=500)
+    first = next(stream)
+    _delete_the_bytes_from_another_connection(path, image_id, dedup=zb._backend._dedup)
+    rest = b"".join(stream)
+    assert first + rest == data
+
+
+def test_a_row_deleted_before_the_first_chunk_raises_storage_error(impl, zb, path):
+    image_id = zb.put(_noise_jpeg())
+    stream = zb.get_stream(image_id, chunk_size=500)  # nothing has been read yet
+    _delete_the_bytes_from_another_connection(path, image_id, dedup=zb._backend._dedup)
+    with pytest.raises(StorageError):
+        next(stream)
+
+
+def test_an_open_stream_does_not_block_writers_in_other_connections(impl, zb, path):
+    image_id = zb.put(_noise_jpeg())
+    stream = zb.get_stream(image_id, chunk_size=500)
+    next(stream)  # a read transaction is now open
+    other = ZeroBucket(backend=SQLiteBackend(path, dedup=zb._backend._dedup))
+    try:
+        # would raise "database is locked" if the open stream blocked writers
+        new_id = other.put(_noise_jpeg(64))
+        assert other.exists(new_id)
+    finally:
+        other.close()
+        stream.close()
+
+
+def test_the_read_transaction_is_released_after_completion_and_after_abandonment(impl, zb, path):
+    data = _noise_jpeg()
+    image_id = zb.put(data)
+    conn = sqlite3.connect(path)
+    try:
+        assert b"".join(zb.get_stream(image_id, chunk_size=700, connection=conn)) == data
+        assert conn.in_transaction is False, "finished stream left a transaction open"
+
+        stream = zb.get_stream(image_id, chunk_size=700, connection=conn)
+        next(stream)
+        stream.close()  # abandoned half way
+        assert conn.in_transaction is False, "abandoned stream left a transaction open"
+    finally:
+        conn.close()
+
+
+def test_a_callers_own_open_transaction_is_used_and_left_open(impl, zb, path):
+    data = _noise_jpeg()
+    image_id = zb.put(data)
+    conn = sqlite3.connect(path)
+    try:
+        conn.execute("BEGIN")
+        assert conn.in_transaction
+        assert b"".join(zb.get_stream(image_id, chunk_size=700, connection=conn)) == data
+        assert conn.in_transaction, "the caller's transaction must not be ended by the stream"
+        conn.rollback()
+    finally:
+        conn.close()
+
+
+def test_two_references_to_one_dedup_blob_each_stream_correctly(impl, path):
+    zb = ZeroBucket(backend=SQLiteBackend(path, dedup=True))
+    try:
+        data = _noise_jpeg()
+        a, b = zb.put(data), zb.put(data)
+        assert b"".join(zb.get_stream(a, chunk_size=333)) == data
+        assert b"".join(zb.get_stream(b, chunk_size=1_000)) == data
+    finally:
+        zb.close()