zodb-pgjsonb 0.1.dev19__tar.gz

zodb_pgjsonb-0.1.dev19/.github/workflows/ci.yaml

@@ -0,0 +1,21 @@
+---
+name: CI
+
+on:
+  push:
+    branches-ignore: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qa:
+    uses: "./.github/workflows/qa.yaml"
+
+  tests:
+    uses: "./.github/workflows/tests.yaml"

zodb_pgjsonb-0.1.dev19/.github/workflows/qa.yaml

@@ -0,0 +1,23 @@
+---
+name: QA
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v6
+
+      - name: Run ruff check
+        run: uvx ruff check .
+
+      - name: Run ruff format check
+        run: uvx ruff format --check .

zodb_pgjsonb-0.1.dev19/.github/workflows/release.yaml

@@ -0,0 +1,80 @@
+---
+name: Build & upload PyPI package
+
+on:
+  push:
+    branches: [main]
+  release:
+    types:
+      - published
+  workflow_dispatch:
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qa:
+    uses: "./.github/workflows/qa.yaml"
+
+  tests:
+    uses: "./.github/workflows/tests.yaml"
+
+  build-package:
+    name: Build & verify package
+    needs:
+      - qa
+      - tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - uses: hynek/build-and-inspect-python-package@v2
+
+  release-test-pypi:
+    name: Publish in-dev package to test.pypi.org
+    environment: release-test-pypi
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs:
+      - build-package
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download packages built by build-and-inspect-python-package
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+
+      - name: Upload package to Test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  release-pypi:
+    name: Publish released package to pypi.org
+    environment: release-pypi
+    if: github.event.action == 'published'
+    runs-on: ubuntu-latest
+    needs:
+      - build-package
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download packages built by build-and-inspect-python-package
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+
+      - name: Upload package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

zodb_pgjsonb-0.1.dev19/.github/workflows/tests.yaml

@@ -0,0 +1,101 @@
+---
+name: Tests
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: zodb
+          POSTGRES_PASSWORD: zodb
+          POSTGRES_DB: zodb_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U zodb"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 5
+
+    env:
+      ZODB_TEST_DSN: "dbname=zodb_test user=zodb password=zodb host=localhost port=5432"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv venv && uv pip install -e ".[test,s3]"
+
+      - name: Run tests with coverage
+        run: uv run pytest --cov --cov-report=
+
+      - name: Rename coverage data
+        run: mv .coverage .coverage.${{ matrix.python-version }}
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-data-${{ matrix.python-version }}
+          path: .coverage.${{ matrix.python-version }}
+          include-hidden-files: true
+          if-no-files-found: ignore
+
+  coverage:
+    name: Combine & check coverage
+    if: always()
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv venv && uv pip install -e ".[test]"
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+
+      - name: Combine coverage & check threshold
+        run: |
+          uv run coverage combine
+          uv run coverage html --skip-covered --skip-empty
+          uv run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+          uv run coverage report
+
+      - name: Upload HTML report if check failed
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}

zodb_pgjsonb-0.1.dev19/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.coverage
+*.egg
+instance/
+uv.lock

zodb_pgjsonb-0.1.dev19/ARCHITECTURE.md

@@ -0,0 +1,269 @@
+# Architecture: ZODB PostgreSQL JSONB Storage
+
+## Vision
+
+A modern, PostgreSQL-only ZODB storage that stores object state as **JSONB** (not pickle/bytea), using **zodb-json-codec** for transparent transcoding. Blobs use tiered storage (small in PG bytea, large in S3 via zodb-s3blobs). Enables SQL queryability of ZODB data while remaining fully transparent to ZODB/Zope/Plone.
+
+## Design Decisions
+
+- **Direction C: Informed Independent** — new independent storage, studying proven algorithms from RelStorage
+- **Separate repo/package** — https://github.com/bluedynamics/zodb-pgjsonb
+- **Both history modes (configurable)** — history-free and history-preserving, like RelStorage
+- **Tiered blob storage** — small blobs in PG bytea, large blobs in S3 (configurable threshold)
+
+---
+
+## Architecture
+
+```
+ZODB.DB
+  └-> PGJsonbStorage (IStorage + IMVCCStorage + IBlobStorage + IStorageUndoable + IStorageIteration + IStorageRestoreable)
+       ├-> zodb-json-codec    (pickle <-> JSON transcoding, Rust)
+       ├-> psycopg3           (sync, pipelined writes via executemany, ConnectionPool)
+       ├-> LISTEN/NOTIFY      (instant invalidation via PG trigger)
+       ├-> Tiered blobs       (PG bytea + optional S3 via zodb-s3blobs)
+       └-> Pure SQL pack/GC   (recursive CTE on pre-extracted refs column)
+```
+
+### What we borrow from RelStorage (algorithms, not code)
+
+- Transaction ID allocation (monotonic, global lock via `pg_advisory_xact_lock`)
+- MVCC semantics (snapshot isolation, `new_instance()` per connection)
+- Pack algorithm concept (mark reachable, sweep unreachable) — reimplemented as pure SQL graph traversal via `refs` column, no object loading needed
+- Conflict resolution via inherited `ConflictResolvingStorage` (data is pickle at boundaries)
+
+### What we design fresh
+
+- Schema optimized for JSONB from day one
+- psycopg3 sync API with pipelined `executemany()` batch writes
+- LISTEN/NOTIFY for instant invalidation
+- Tiered blob storage with configurable threshold
+- Configurable history modes (history-free / history-preserving)
+- SQL query API as first-class citizen
+
+---
+
+## Schema Design
+
+### History-Free Mode (default, recommended for Plone)
+
+```sql
+CREATE TABLE transaction_log (
+    tid         BIGINT PRIMARY KEY,
+    username    TEXT DEFAULT '',
+    description TEXT DEFAULT '',
+    extension   BYTEA DEFAULT ''
+);
+
+CREATE TABLE object_state (
+    zoid        BIGINT PRIMARY KEY,
+    tid         BIGINT NOT NULL REFERENCES transaction_log(tid),
+    class_mod   TEXT NOT NULL,
+    class_name  TEXT NOT NULL,
+    state       JSONB,
+    state_size  INTEGER NOT NULL,
+    refs        BIGINT[] NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE blob_state (
+    zoid        BIGINT NOT NULL,
+    tid         BIGINT NOT NULL,
+    blob_size   BIGINT NOT NULL,
+    data        BYTEA,
+    s3_key      TEXT,
+    PRIMARY KEY (zoid, tid)
+);
+
+CREATE INDEX idx_object_class ON object_state (class_mod, class_name);
+CREATE INDEX idx_object_state_gin ON object_state USING gin (state jsonb_path_ops);
+CREATE INDEX idx_object_refs ON object_state USING gin (refs);
+```
+
+### History-Preserving Mode (adds history tables)
+
+```sql
+CREATE TABLE object_history (
+    zoid        BIGINT NOT NULL,
+    tid         BIGINT NOT NULL,
+    class_mod   TEXT NOT NULL,
+    class_name  TEXT NOT NULL,
+    state       JSONB,
+    state_size  INTEGER NOT NULL,
+    refs        BIGINT[] NOT NULL DEFAULT '{}',
+    PRIMARY KEY (zoid, tid)
+);
+
+CREATE TABLE blob_history (
+    zoid        BIGINT NOT NULL,
+    tid         BIGINT NOT NULL,
+    blob_size   BIGINT NOT NULL,
+    data        BYTEA,
+    s3_key      TEXT,
+    PRIMARY KEY (zoid, tid)
+);
+
+CREATE TABLE pack_state (
+    zoid        BIGINT PRIMARY KEY,
+    tid         BIGINT NOT NULL
+);
+```
+
+### Schema Notes
+
+- **Class columns inline** (not normalized FK): avoids JOIN on the hot path (`load()`), class strings compress well via TOAST
+- **`state` is `@s` only**: the `@cls` marker is extracted into `class_mod`/`class_name` columns, saving JSONB space and enabling efficient class-based queries
+- **`state_size`**: original pickle size, useful for monitoring
+- **Advisory locks** for commit serialization: `pg_advisory_xact_lock(0)`, released automatically on transaction end
+- **GIN index with `jsonb_path_ops`**: enables `@>` containment queries efficiently
+- **`refs` column**: pre-extracted persistent reference OIDs, enables pure SQL pack/GC
+
+### Pack via Pure SQL Graph Traversal
+
+Traditional ZODB pack loads and unpickles every object to extract references. PGJsonbStorage extracts `@ref` OIDs at `store()` time into the `refs` column, enabling pack as a single recursive SQL query:
+
+```sql
+WITH RECURSIVE reachable AS (
+    SELECT zoid FROM object_state WHERE zoid = 0
+    UNION
+    SELECT unnest(o.refs)
+    FROM object_state o
+    JOIN reachable r ON o.zoid = r.zoid
+)
+DELETE FROM object_state
+WHERE zoid NOT IN (SELECT zoid FROM reachable);
+```
+
+Result: 15-28x faster than RelStorage for pack operations.
+
+---
+
+## Tiered Blob Storage
+
+| Mode     | Config                     | Where blobs are stored        |
+|----------|----------------------------|-------------------------------|
+| PG-only  | No S3 keys set             | All blobs in PostgreSQL bytea |
+| Tiered   | S3 keys + blob-threshold   | Small in PG, large in S3     |
+| S3-only  | S3 keys + blob-threshold=0 | All blobs in S3               |
+
+S3 tiering uses `zodb-s3blobs` (`S3Client` + `S3BlobCache`) as optional dependency.
+
+---
+
+## Transcoding Integration
+
+### Store Path (ZODB -> PostgreSQL)
+
+```
+pickle bytes -> zodb_json_codec.decode_zodb_record()
+  -> {"@cls": [mod, name], "@s": {state...}}
+  -> extract class_mod, class_name, state, refs
+  -> INSERT INTO object_state (JSONB)
+```
+
+### Load Path (PostgreSQL -> ZODB)
+
+```
+SELECT class_mod, class_name, state FROM object_state
+  -> reconstruct {"@cls": [mod, name], "@s": {state...}}
+  -> zodb_json_codec.encode_zodb_record()
+  -> pickle bytes
+```
+
+### Conflict Resolution
+
+Data is pickle bytes at the ZODB boundary. `ConflictResolvingStorage.tryToResolveConflict()` receives pickle, unpickles, calls `_p_resolveConflict()`, re-pickles. The default identity transform hooks are correct since all data at the resolution boundary is already pickle.
+
+---
+
+## MVCC Architecture
+
+```
+PGJsonbStorage (main, factory)
+  ├── implements IMVCCStorage
+  ├── owns: schema, OID allocation, DSN, connection pool
+  ├── new_instance() -> PGJsonbStorageInstance
+  └── close() closes pool + main connection
+
+PGJsonbStorageInstance (per-connection)
+  ├── borrows PG connection from pool (autocommit=True)
+  ├── REPEATABLE READ snapshots for consistent reads
+  ├── poll_invalidations() via SQL query
+  ├── store/tpc_begin/vote/finish/abort (write path)
+  └── release() returns connection to pool
+```
+
+---
+
+## Migration via zodbconvert
+
+PGJsonbStorage implements `IStorageRestoreable` (`restore()`, `restoreBlob()`), making it compatible with `zodbconvert` out of the box:
+
+```ini
+<source>
+    <relstorage>
+        <postgresql>
+            dsn dbname='zodb_old'
+        </postgresql>
+    </relstorage>
+</source>
+<destination>
+    %import zodb_pgjsonb
+    <pgjsonb>
+        dsn dbname='zodb_new'
+    </pgjsonb>
+</destination>
+```
+
+---
+
+## Package Structure
+
+```
+zodb-pgjsonb/
+├── src/zodb_pgjsonb/
+│   ├── __init__.py           # public API
+│   ├── storage.py            # PGJsonbStorage + PGJsonbStorageInstance (core)
+│   ├── schema.py             # DDL definitions
+│   ├── objects.py            # ObjectStore (JSONB CRUD)
+│   ├── invalidation.py       # LISTEN/NOTIFY channel
+│   ├── blobs.py              # BlobManager (tiered PG/S3)
+│   ├── packer.py             # Pack/GC (pure SQL)
+│   ├── interfaces.py         # Zope interface declarations
+│   ├── config.py             # ZConfig factory
+│   └── component.xml         # ZConfig schema definition
+├── tests/
+│   ├── test_storage.py       # Core IStorage tests
+│   ├── test_mvcc.py          # MVCC / poll_invalidations
+│   ├── test_history.py       # History-preserving mode + undo
+│   ├── test_blobs.py         # Blob storage (PG + S3)
+│   ├── test_conflict.py      # Conflict resolution (BTrees)
+│   ├── test_conformance.py   # ZODB conformance test suite
+│   ├── test_iteration.py     # IStorageIteration
+│   ├── test_roundtrip.py     # Codec roundtrip verification
+│   └── test_stress.py        # Concurrent write stress tests
+├── benchmarks/
+│   └── bench.py              # Performance benchmarks vs RelStorage
+├── example/                  # Docker Compose demo setup
+├── ARCHITECTURE.md           # This file
+├── BENCHMARKS.md             # Performance results
+├── pyproject.toml
+└── README.md
+```
+
+---
+
+## Comparison: RelStorage vs PGJsonbStorage
+
+| Feature | RelStorage | PGJsonbStorage |
+|---------|-----------|----------------|
+| Storage format | pickle/bytea | JSONB |
+| SQL queryable | No | Yes (GIN indexes, jsonpath) |
+| Databases | PG, MySQL, Oracle, SQLite | PostgreSQL only |
+| Python driver | psycopg2 | psycopg3 (pipelined writes, pool) |
+| Invalidation | Polling (1-5s latency) | LISTEN/NOTIFY (~1ms) |
+| Blob storage | Filesystem or cloud (separate) | Tiered PG + S3 (integrated) |
+| History modes | Both | Both (configurable) |
+| Conflict resolution | Built-in | Via inherited ConflictResolvingStorage |
+| Pack algorithm | Load+unpickle every object | Pure SQL graph traversal via `refs` column |
+| Codec dependency | None | zodb-json-codec (Rust) |
+| Maturity | 15+ years | New |

zodb_pgjsonb-0.1.dev19/BENCHMARKS.md

@@ -0,0 +1,126 @@
+# Performance Benchmarks
+
+Comparison of `PGJsonbStorage` vs `RelStorage` (both using PostgreSQL)
+for ZODB storage operations at multiple abstraction levels.
+
+Measured on: 2026-02-08
+Python: 3.13.9, PostgreSQL: 17, 100 iterations, 10 warmup
+Host: localhost, Docker-containerized PostgreSQL
+
+## Context
+
+PGJsonbStorage stores object state as **JSONB** (enabling SQL queries on ZODB
+data), while RelStorage stores **pickle bytes** as bytea. This fundamental
+difference affects the performance profile:
+
+- **Writes**: PGJsonbStorage transcodes pickle to JSONB via `zodb-json-codec`
+  (Rust), then PostgreSQL indexes the JSONB. RelStorage writes raw bytes.
+  Batch writes use psycopg3's `executemany()` with pipeline mode for
+  pipelined SQL (single network round-trip per batch).
+- **Reads (raw)**: Both storages use a local in-memory LRU cache
+  (`cache_local_mb`) that serves hot objects from RAM. Cache misses hit
+  PostgreSQL — PGJsonbStorage additionally transcodes JSONB back to pickle.
+- **Reads (ZODB)**: ZODB's object cache handles >95% of reads in production,
+  making the raw storage read speed largely irrelevant.
+- **Pack/GC**: PGJsonbStorage uses pure SQL graph traversal via pre-extracted
+  `refs` column (recursive CTE, no data leaves PostgreSQL). RelStorage must
+  load and unpickle every object to extract references.
+
+## Storage API (raw store/load)
+
+| Operation | PGJsonb | RelStorage | Comparison |
+|---|---|---|---|
+| store single | 5.2 ms | 8.4 ms | **1.6x faster** |
+| store batch 10 | 8.4 ms | 8.5 ms | **on par** |
+| store batch 100 | 12.4 ms | 10.4 ms | 1.2x slower |
+| load single | 1 us | 1 us | **1.4x faster** |
+| load batch 100 | 21 us | 96 us | **4.6x faster** |
+
+Single stores are faster because PGJsonbStorage has a simpler 2PC path (direct
+SQL, no OID/TID tracking tables). Batch stores use pipelined `executemany()`
+— transcoding is <1% of batch time (0.14 ms for 100 objects via
+`decode_zodb_record_for_pg`), the remaining cost is PostgreSQL JSONB indexing
+overhead vs RelStorage's raw bytea writes. Both storages serve hot reads from
+their local LRU cache — PGJsonbStorage's cache is slightly faster due to
+simpler lookup logic.
+
+## ZODB.DB (through object cache)
+
+| Operation | PGJsonb | RelStorage | Comparison |
+|---|---|---|---|
+| write simple | 8.9 ms | 8.7 ms | on par |
+| write btree | 7.7 ms | 10.8 ms | **1.4x faster** |
+| read | 3 us | 3 us | on par |
+| connection cycle | 224 us | 240 us | **1.1x faster** |
+| write batch 10 | 12.6 ms | 11.2 ms | 1.1x slower |
+
+Through ZODB.DB, ZODB's object cache handles the hot read path. Writes are
+on par. Both storages use connection pooling (`psycopg_pool`
+for PGJsonbStorage, built-in for RelStorage). Connection cycle overhead is
+on par thanks to pool pre-warming (`pool-size=1` default).
+
+## Pack / GC
+
+| Objects | PGJsonb | RelStorage | Comparison |
+|---|---|---|---|
+| 100 | 14.0 ms | 202.5 ms | **14.5x faster** |
+| 1,000 | 8.3 ms | 236.8 ms | **28.4x faster** |
+| 10,000 | 23.5 ms | 604.7 ms | **25.7x faster** |
+
+Pack is the standout advantage. PGJsonbStorage's pure SQL graph traversal
+via the pre-extracted `refs` column (recursive CTE) runs entirely inside
+PostgreSQL — no objects are loaded, no Python unpickling occurs. RelStorage
+must load and unpickle every object to discover references via `referencesf()`.
+The advantage grows with database size.
+
+## Plone Application (50 documents)
+
+| Operation | PGJsonb | RelStorage | Comparison |
+|---|---|---|---|
+| site creation | 1.05 s | 1.06 s | on par |
+| content create/doc | 27.6 ms | 27.2 ms | on par |
+| catalog query | 187 us | 182 us | on par |
+| content modify/doc | 6.6 ms | 7.0 ms | **1.1x faster** |
+
+At the Plone application level, both storage backends are **on par**. The
+storage layer accounts for a small fraction of total request time, which
+is dominated by Plone framework overhead (catalog indexing, security checks,
+event subscribers, ZCML lookups).
+
+This confirms that PGJsonbStorage can serve as a drop-in replacement for
+RelStorage in Plone deployments, trading minimal overhead for **SQL
+queryability of all ZODB data via JSONB**.
+
+## Analysis
+
+**Strengths:**
+- Single-object writes 1.4-1.6x faster (simpler 2PC)
+- Raw loads 1.4-4.6x faster (both cached, simpler lookup)
+- Pack/GC 15-28x faster (pure SQL, no object loading)
+- Plone-level performance on par with RelStorage
+- All ZODB data queryable via SQL/JSONB (unique to PGJsonbStorage)
+
+**Trade-offs:**
+- Batch store 100 is 1.2x slower (JSONB indexing overhead vs raw bytea)
+
+## Running Benchmarks
+
+```bash
+cd sources/zodb-pgjsonb
+
+# Requires PostgreSQL on localhost:5433 with benchmark databases:
+#   createdb -h localhost -p 5433 -U zodb zodb_bench_pgjsonb
+#   createdb -h localhost -p 5433 -U zodb zodb_bench_relstorage
+
+# Install benchmark dependencies
+uv pip install -e ".[bench]"
+
+# Individual benchmark categories
+python benchmarks/bench.py storage --iterations 100
+python benchmarks/bench.py zodb --iterations 100
+python benchmarks/bench.py pack
+python benchmarks/bench.py plone --docs 50
+
+# All benchmarks with JSON export
+python benchmarks/bench.py all --output benchmarks/results.json --format both
+```