zodb-convert 0.0.0.dev4__tar.gz

zodb_convert-0.0.0.dev4/.github/workflows/ci.yaml

@@ -0,0 +1,21 @@
+---
+name: CI
+
+on:
+  push:
+    branches-ignore: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qa:
+    uses: "./.github/workflows/qa.yaml"
+
+  tests:
+    uses: "./.github/workflows/tests.yaml"

zodb_convert-0.0.0.dev4/.github/workflows/qa.yaml

@@ -0,0 +1,23 @@
+---
+name: QA
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v6
+
+      - name: Run ruff check
+        run: uvx ruff check .
+
+      - name: Run ruff format check
+        run: uvx ruff format --check .

zodb_convert-0.0.0.dev4/.github/workflows/release.yaml

@@ -0,0 +1,80 @@
+---
+name: Build & upload PyPI package
+
+on:
+  push:
+    branches: [main]
+  release:
+    types:
+      - published
+  workflow_dispatch:
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qa:
+    uses: "./.github/workflows/qa.yaml"
+
+  tests:
+    uses: "./.github/workflows/tests.yaml"
+
+  build-package:
+    name: Build & verify package
+    needs:
+      - qa
+      - tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - uses: hynek/build-and-inspect-python-package@v2
+
+  release-test-pypi:
+    name: Publish in-dev package to test.pypi.org
+    environment: release-test-pypi
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs:
+      - build-package
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download packages built by build-and-inspect-python-package
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+
+      - name: Upload package to Test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  release-pypi:
+    name: Publish released package to pypi.org
+    environment: release-pypi
+    if: github.event.action == 'published'
+    runs-on: ubuntu-latest
+    needs:
+      - build-package
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download packages built by build-and-inspect-python-package
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+
+      - name: Upload package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

zodb_convert-0.0.0.dev4/.github/workflows/tests.yaml

@@ -0,0 +1,83 @@
+---
+name: Tests
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv venv && uv pip install -e ".[test]"
+
+      - name: Run tests with coverage
+        run: uv run pytest --cov --cov-report=
+
+      - name: Rename coverage data
+        run: mv .coverage .coverage.${{ matrix.python-version }}
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-data-${{ matrix.python-version }}
+          path: .coverage.${{ matrix.python-version }}
+          include-hidden-files: true
+          if-no-files-found: ignore
+
+  coverage:
+    name: Combine & check coverage
+    if: always()
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv venv && uv pip install -e ".[test]"
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+
+      - name: Combine coverage & check threshold
+        run: |
+          uv run coverage combine
+          uv run coverage html --skip-covered --skip-empty
+          uv run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+          uv run coverage report
+
+      - name: Upload HTML report if check failed
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}

zodb_convert-0.0.0.dev4/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+.venv/
+.pytest_cache/
+.coverage
+*.egg

zodb_convert-0.0.0.dev4/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files

zodb_convert-0.0.0.dev4/CHANGES.md

@@ -0,0 +1,23 @@
+# Changelog
+
+## 1.0.0a1 (unreleased)
+
+Initial release. Derived from
+[RelStorage's zodbconvert](https://github.com/zodb/relstorage)
+(Copyright Zope Foundation and Contributors, ZPL-2.1).
+
+### Differences from RelStorage's zodbconvert
+
+- **Standalone package**: no RelStorage dependency -- works with any
+  ZODB-compatible storage (FileStorage, RelStorage, zodb-pgjsonb, ZEO, ...).
+- **zope.conf extraction**: `--source-zope-conf` / `--dest-zope-conf` flags
+  extract storage configuration from existing Zope config files, no need to
+  write a separate conversion config.
+- **Mixed mode**: combine a traditional config file with zope.conf extraction
+  (e.g. destination from config file, source from zope.conf).
+- **Enhanced progress reporting**: multi-tier output with per-transaction
+  logging for small conversions, interval-based updates with throughput and
+  ETA for large ones, and a summary at completion.
+- **Incremental copy**: `--incremental` resumes from the last transaction in
+  the destination storage.
+- **Dry-run mode**: `--dry-run` previews what would be copied.

zodb_convert-0.0.0.dev4/LICENSE.txt

@@ -0,0 +1,43 @@
+Zope Public License (ZPL) Version 2.1
+
+A copyright notice accompanies this license document that identifies the
+copyright holders.
+
+This license has been certified as open source. It has also been designated as
+GPL compatible by the Free Software Foundation (FSF).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions in source code must retain the accompanying copyright
+   notice, this list of conditions, and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the accompanying copyright
+   notice, this list of conditions, and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Names of the copyright holders must not be used to endorse or promote
+   products derived from this software without prior written permission from
+   the copyright holders.
+
+4. The right to distribute this software or to use it for any purpose does not
+   give you the right to use Servicemarks (sm) or Trademarks (tm) of the
+   copyright holders. Use of them is covered by separate agreement with the
+   copyright holders.
+
+5. If any files are modified, you must cause the modified files to carry
+   prominent notices stating that you changed the files and the date of any
+   change.
+
+Disclaimer
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

zodb_convert-0.0.0.dev4/PKG-INFO

@@ -0,0 +1,139 @@
+Metadata-Version: 2.4
+Name: zodb-convert
+Version: 0.0.0.dev4
+Summary: Generic ZODB storage conversion tool
+Project-URL: Homepage, https://github.com/bluedynamics/zodb-convert
+Project-URL: Repository, https://github.com/bluedynamics/zodb-convert
+Project-URL: Issues, https://github.com/bluedynamics/zodb-convert/issues
+License-Expression: ZPL-2.1
+License-File: LICENSE.txt
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: ZODB
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Zope Public License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Requires-Python: >=3.12
+Requires-Dist: transaction
+Requires-Dist: zconfig
+Requires-Dist: zodb>=5.8
+Requires-Dist: zope-interface
+Provides-Extra: test
+Requires-Dist: coverage[toml]>=7.0; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytest-cov; extra == 'test'
+Description-Content-Type: text/markdown
+
+# zodb-convert
+
+Generic ZODB storage conversion tool. Copies data between any two ZODB-compatible storages.
+
+Derived from [RelStorage's zodbconvert](https://github.com/zodb/relstorage) by the
+Zope Foundation and Contributors, extracted as a standalone generic tool.
+
+## Installation
+
+```bash
+pip install zodb-convert
+```
+
+To convert between specific storage types, install their packages too:
+
+```bash
+pip install zodb-convert ZODB RelStorage  # FileStorage ↔ RelStorage
+pip install zodb-convert ZODB zodb-pgjsonb  # FileStorage ↔ PGJsonb
+```
+
+## Usage
+
+### Traditional config file
+
+Create a config file (`convert.conf`):
+
+```
+<filestorage source>
+    path /data/Data.fs
+    blob-dir /data/blobs
+</filestorage>
+
+<filestorage destination>
+    path /backup/Data.fs
+    blob-dir /backup/blobs
+</filestorage>
+```
+
+For storages that need blob wrapping:
+
+```
+<blobstorage source>
+    blob-dir /data/blobs
+    <filestorage>
+        path /data/Data.fs
+    </filestorage>
+</blobstorage>
+
+<filestorage destination>
+    path /backup/Data.fs
+    blob-dir /backup/blobs
+</filestorage>
+```
+
+For third-party storages, use `%import`:
+
+```
+%import relstorage
+
+<relstorage source>
+    <postgresql>
+        dsn dbname=zodb user=zodb
+    </postgresql>
+</relstorage>
+
+<filestorage destination>
+    path /backup/Data.fs
+</filestorage>
+```
+
+Run:
+
+```bash
+zodb-convert convert.conf
+```
+
+### Using existing zope.conf files
+
+Extract storage configuration directly from Zope config files:
+
+```bash
+zodb-convert --source-zope-conf /old/etc/zope.conf --dest-zope-conf /new/etc/zope.conf
+```
+
+Specify which database to use (defaults to "main"):
+
+```bash
+zodb-convert --source-zope-conf zope.conf --source-db main \
+             --dest-zope-conf other.conf --dest-db catalog
+```
+
+### Mixed mode
+
+Combine traditional config with zope.conf extraction:
+
+```bash
+zodb-convert convert.conf --source-zope-conf /old/etc/zope.conf
+```
+
+Where `convert.conf` contains only the destination storage section.
+
+### Options
+
+- `--dry-run` — show what would be copied without making changes
+- `--incremental` — resume from the last transaction in the destination
+- `-v` / `--verbose` — increase verbosity (`-v` for INFO, `-vv` for DEBUG)
+
+## License
+
+ZPL-2.1 — see [LICENSE.txt](LICENSE.txt)

zodb_convert-0.0.0.dev4/README.md

@@ -0,0 +1,110 @@
+# zodb-convert
+
+Generic ZODB storage conversion tool. Copies data between any two ZODB-compatible storages.
+
+Derived from [RelStorage's zodbconvert](https://github.com/zodb/relstorage) by the
+Zope Foundation and Contributors, extracted as a standalone generic tool.
+
+## Installation
+
+```bash
+pip install zodb-convert
+```
+
+To convert between specific storage types, install their packages too:
+
+```bash
+pip install zodb-convert ZODB RelStorage  # FileStorage ↔ RelStorage
+pip install zodb-convert ZODB zodb-pgjsonb  # FileStorage ↔ PGJsonb
+```
+
+## Usage
+
+### Traditional config file
+
+Create a config file (`convert.conf`):
+
+```
+<filestorage source>
+    path /data/Data.fs
+    blob-dir /data/blobs
+</filestorage>
+
+<filestorage destination>
+    path /backup/Data.fs
+    blob-dir /backup/blobs
+</filestorage>
+```
+
+For storages that need blob wrapping:
+
+```
+<blobstorage source>
+    blob-dir /data/blobs
+    <filestorage>
+        path /data/Data.fs
+    </filestorage>
+</blobstorage>
+
+<filestorage destination>
+    path /backup/Data.fs
+    blob-dir /backup/blobs
+</filestorage>
+```
+
+For third-party storages, use `%import`:
+
+```
+%import relstorage
+
+<relstorage source>
+    <postgresql>
+        dsn dbname=zodb user=zodb
+    </postgresql>
+</relstorage>
+
+<filestorage destination>
+    path /backup/Data.fs
+</filestorage>
+```
+
+Run:
+
+```bash
+zodb-convert convert.conf
+```
+
+### Using existing zope.conf files
+
+Extract storage configuration directly from Zope config files:
+
+```bash
+zodb-convert --source-zope-conf /old/etc/zope.conf --dest-zope-conf /new/etc/zope.conf
+```
+
+Specify which database to use (defaults to "main"):
+
+```bash
+zodb-convert --source-zope-conf zope.conf --source-db main \
+             --dest-zope-conf other.conf --dest-db catalog
+```
+
+### Mixed mode
+
+Combine traditional config with zope.conf extraction:
+
+```bash
+zodb-convert convert.conf --source-zope-conf /old/etc/zope.conf
+```
+
+Where `convert.conf` contains only the destination storage section.
+
+### Options
+
+- `--dry-run` — show what would be copied without making changes
+- `--incremental` — resume from the last transaction in the destination
+- `-v` / `--verbose` — increase verbosity (`-v` for INFO, `-vv` for DEBUG)
+
+## License
+
+ZPL-2.1 — see [LICENSE.txt](LICENSE.txt)

zodb_convert-0.0.0.dev4/pyproject.toml

@@ -0,0 +1,106 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "zodb-convert"
+dynamic = ["version"]
+description = "Generic ZODB storage conversion tool"
+readme = "README.md"
+license = "ZPL-2.1"
+requires-python = ">=3.12"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Zope Public License",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Database",
+    "Framework :: ZODB",
+]
+dependencies = [
+    "ZODB>=5.8",
+    "ZConfig",
+    "zope.interface",
+    "transaction",
+]
+
+[project.urls]
+Homepage = "https://github.com/bluedynamics/zodb-convert"
+Repository = "https://github.com/bluedynamics/zodb-convert"
+Issues = "https://github.com/bluedynamics/zodb-convert/issues"
+
+[project.scripts]
+zodb-convert = "zodb_convert.cli:main"
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-cov",
+    "coverage[toml]>=7.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/zodb_convert"]
+
+[tool.hatch.version]
+source = "vcs"
+root = "../.."
+raw-options = { local_scheme = "no-local-version" }
+fallback-version = "0.0.0.dev0"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.coverage.run]
+parallel = true
+source = ["zodb_convert"]
+relative_files = true
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 90
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+]
+
+[tool.ruff]
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+    "A",
+    "B",
+    "C4",
+    "E",
+    "F",
+    "I",
+    "RUF",
+    "SIM",
+    "T20",
+    "UP",
+    "W",
+]
+ignore = [
+    "E501",
+    "RUF001",
+    "RUF002",
+]
+
+[tool.ruff.lint.isort]
+force-single-line = true
+from-first = true
+lines-after-imports = 2
+lines-between-types = 1
+no-sections = true
+order-by-type = false
+
+[tool.ruff.lint.pycodestyle]
+max-line-length = 120
+max-doc-length = 120
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["T201"]