duckdb-safe-snapshot 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- duckdb_safe_snapshot-0.1.0/.github/workflows/ci.yml +26 -0
- duckdb_safe_snapshot-0.1.0/.github/workflows/release.yml +60 -0
- duckdb_safe_snapshot-0.1.0/.gitignore +6 -0
- duckdb_safe_snapshot-0.1.0/AGENT_INTEGRATION.md +24 -0
- duckdb_safe_snapshot-0.1.0/CONTRIBUTING.md +12 -0
- duckdb_safe_snapshot-0.1.0/LICENSE +182 -0
- duckdb_safe_snapshot-0.1.0/NOTICE +5 -0
- duckdb_safe_snapshot-0.1.0/PKG-INFO +157 -0
- duckdb_safe_snapshot-0.1.0/README.md +135 -0
- duckdb_safe_snapshot-0.1.0/RELEASING.md +7 -0
- duckdb_safe_snapshot-0.1.0/SECURITY.md +8 -0
- duckdb_safe_snapshot-0.1.0/docs/assets/synthetic-snapshot-result.svg +12 -0
- duckdb_safe_snapshot-0.1.0/pyproject.toml +34 -0
- duckdb_safe_snapshot-0.1.0/src/duckdb_safe_snapshot/__init__.py +6 -0
- duckdb_safe_snapshot-0.1.0/src/duckdb_safe_snapshot/cli.py +77 -0
- duckdb_safe_snapshot-0.1.0/src/duckdb_safe_snapshot/core.py +481 -0
- duckdb_safe_snapshot-0.1.0/tests/test_inherited_lock.py +34 -0
- duckdb_safe_snapshot-0.1.0/tests/test_snapshot.py +308 -0
- duckdb_safe_snapshot-0.1.0/tools/pypi_release_state.py +28 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
push:
|
|
6
|
+
branches: [main]
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
runs-on: ubuntu-24.04
|
|
14
|
+
strategy:
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
19
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python-version }}
|
|
22
|
+
- run: python -m pip install --upgrade pip build pytest duckdb==1.5.5
|
|
23
|
+
- run: python -m build
|
|
24
|
+
- run: python -m venv consumer
|
|
25
|
+
- run: consumer/bin/python -m pip install dist/*.whl pytest duckdb==1.5.5
|
|
26
|
+
- run: consumer/bin/python -m pytest tests
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
publish:
|
|
12
|
+
runs-on: ubuntu-24.04
|
|
13
|
+
environment:
|
|
14
|
+
name: pypi
|
|
15
|
+
url: https://pypi.org/p/duckdb-safe-snapshot
|
|
16
|
+
permissions: {contents: read, id-token: write}
|
|
17
|
+
env:
|
|
18
|
+
RELEASE_TAG: ${{ github.ref_name }}
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
21
|
+
with:
|
|
22
|
+
ref: ${{ github.sha }}
|
|
23
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
24
|
+
with: {python-version: '3.12'}
|
|
25
|
+
- run: |
|
|
26
|
+
set -euo pipefail
|
|
27
|
+
test "$RELEASE_TAG" = "v$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
|
|
28
|
+
python -m pip install --upgrade pip build pytest duckdb==1.5.5
|
|
29
|
+
python -m build
|
|
30
|
+
python -m venv consumer
|
|
31
|
+
consumer/bin/python -m pip install dist/*.whl pytest duckdb==1.5.5
|
|
32
|
+
consumer/bin/python -m pytest tests
|
|
33
|
+
- id: existing
|
|
34
|
+
env:
|
|
35
|
+
DIST_DIR: dist
|
|
36
|
+
PYPI_PACKAGE: duckdb-safe-snapshot
|
|
37
|
+
PYPI_VERSION: ${{ github.ref_name }}
|
|
38
|
+
run: python tools/pypi_release_state.py
|
|
39
|
+
- if: steps.existing.outputs.state == 'publish'
|
|
40
|
+
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
41
|
+
with: {skip-existing: true}
|
|
42
|
+
- env:
|
|
43
|
+
DIST_DIR: dist
|
|
44
|
+
MODE: post
|
|
45
|
+
PYPI_PACKAGE: duckdb-safe-snapshot
|
|
46
|
+
PYPI_VERSION: ${{ github.ref_name }}
|
|
47
|
+
run: python tools/pypi_release_state.py
|
|
48
|
+
release:
|
|
49
|
+
needs: publish
|
|
50
|
+
runs-on: ubuntu-24.04
|
|
51
|
+
permissions: {contents: write}
|
|
52
|
+
env:
|
|
53
|
+
GH_TOKEN: ${{ github.token }}
|
|
54
|
+
GH_REPO: ${{ github.repository }}
|
|
55
|
+
RELEASE_TAG: ${{ github.ref_name }}
|
|
56
|
+
steps:
|
|
57
|
+
- run: |
|
|
58
|
+
gh release view "$RELEASE_TAG" >/dev/null 2>&1 && exit 0
|
|
59
|
+
if [[ "$RELEASE_TAG" == *-* ]]; then gh release create "$RELEASE_TAG" --verify-tag --generate-notes --prerelease
|
|
60
|
+
else gh release create "$RELEASE_TAG" --verify-tag --generate-notes; fi
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Agent integration guide
|
|
2
|
+
|
|
3
|
+
`duckdb-safe-snapshot` creates and verifies lock-coordinated DuckDB database
|
|
4
|
+
and WAL snapshot sets on Linux. It requires an external `flock` path that every
|
|
5
|
+
participating writer acquires before modifying either source file.
|
|
6
|
+
|
|
7
|
+
Install a pinned release with `python -m pip install duckdb-safe-snapshot==0.1.0`.
|
|
8
|
+
Add it to the consumer's Python dependency file and lockfile, then configure an
|
|
9
|
+
explicit `Config` with absolute database, WAL, lock, backup, and state paths plus
|
|
10
|
+
source/lock/snapshot owner UIDs and database/WAL artifact names. Application
|
|
11
|
+
scheduling, service ordering, ACL setup, backup retention policy, and recovery
|
|
12
|
+
remain consumer-owned policy.
|
|
13
|
+
|
|
14
|
+
Before integration, inspect the consumer's writers and confirm each takes the
|
|
15
|
+
same lock. Use a disposable DuckDB database to run `create_snapshot(config)` and
|
|
16
|
+
`verify_latest(config)`, then copy the database and optional WAL artifacts to a
|
|
17
|
+
recovery filename and reopen it with DuckDB. Verify imports resolve from the
|
|
18
|
+
installed package rather than a source checkout.
|
|
19
|
+
|
|
20
|
+
To undo a first migration, restore the consumer's previous implementation or
|
|
21
|
+
commit, dependency/lockfile, and its tested scheduling configuration. Do not
|
|
22
|
+
delete recovery sets as part of rollback. Report unsupported filesystem behavior
|
|
23
|
+
or a failed verification with synthetic, non-secret diagnostics; use
|
|
24
|
+
[SECURITY.md](SECURITY.md) for a vulnerability.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Use Python 3.10–3.12 on Linux. Install test dependencies in an isolated
|
|
4
|
+
environment, then run:
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
python -m pytest
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Tests create synthetic DuckDB databases and snapshot directories only. Do not
|
|
11
|
+
add real databases, production paths, credentials, or provider data. By
|
|
12
|
+
submitting a contribution, you agree that it may be distributed under Apache-2.0.
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
124
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
125
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
126
|
+
this License, without any additional terms or conditions.
|
|
127
|
+
|
|
128
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
129
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
130
|
+
except as required for reasonable and customary use in describing the
|
|
131
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
132
|
+
|
|
133
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
134
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
135
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
136
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
137
|
+
implied, including, without limitation, any warranties or conditions
|
|
138
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
139
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
140
|
+
appropriateness of using or redistributing the Work and assume any
|
|
141
|
+
risks associated with Your exercise of permissions under this License.
|
|
142
|
+
|
|
143
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
144
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
145
|
+
unless required by applicable law (such as deliberate and grossly
|
|
146
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
147
|
+
liable to You for damages, including any direct, indirect, special,
|
|
148
|
+
incidental, or consequential damages of any character arising as a
|
|
149
|
+
result of this License or out of the use or inability to use the
|
|
150
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
151
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
152
|
+
other commercial damages or losses), even if such Contributor
|
|
153
|
+
has been advised of the possibility of such damages.
|
|
154
|
+
|
|
155
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
156
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
157
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
158
|
+
or other liability obligations and/or rights consistent with this
|
|
159
|
+
License. However, in accepting such obligations, You may act only
|
|
160
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
161
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
162
|
+
defend, and hold each Contributor harmless for any liability
|
|
163
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
164
|
+
of your accepting any such warranty or additional liability.
|
|
165
|
+
|
|
166
|
+
END OF TERMS AND CONDITIONS
|
|
167
|
+
|
|
168
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
169
|
+
|
|
170
|
+
Copyright 2026 Sid Kalla
|
|
171
|
+
|
|
172
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
173
|
+
you may not use this file except in compliance with the License.
|
|
174
|
+
You may obtain a copy of the License at
|
|
175
|
+
|
|
176
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
177
|
+
|
|
178
|
+
Unless required by applicable law or agreed to in writing, software
|
|
179
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
180
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
181
|
+
See the License for the specific language governing permissions and
|
|
182
|
+
limitations under the License.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: duckdb-safe-snapshot
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lock-coordinated, checksummed DuckDB database and WAL snapshots
|
|
5
|
+
Project-URL: Homepage, https://github.com/sidko/duckdb-safe-snapshot
|
|
6
|
+
Project-URL: Issues, https://github.com/sidko/duckdb-safe-snapshot/issues
|
|
7
|
+
Author: Sid Kalla
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: NOTICE
|
|
11
|
+
Keywords: backup,duckdb,snapshot
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# duckdb-safe-snapshot
|
|
24
|
+
|
|
25
|
+
Create private, checksummed snapshot sets of a DuckDB database and its WAL while
|
|
26
|
+
cooperating writers share one external file lock. It is a small Linux library
|
|
27
|
+
and CLI for applications that own their own scheduling and recovery process.
|
|
28
|
+
|
|
29
|
+

|
|
30
|
+
|
|
31
|
+
The image is an illustrative static terminal transcript with synthetic values;
|
|
32
|
+
it is not captured output from a snapshot run.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
python -m pip install duckdb-safe-snapshot==0.1.0
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Every path and owner policy is explicit; the package has no application,
|
|
39
|
+
account, service, or production-path defaults.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
duckdb-safe-snapshot \
|
|
43
|
+
--database /var/lib/example/app.duckdb \
|
|
44
|
+
--wal /var/lib/example/app.duckdb.wal \
|
|
45
|
+
--lock /run/example/writer.lock \
|
|
46
|
+
--backup-root /var/backups/example/snapshots \
|
|
47
|
+
--state-root /var/lib/example/snapshot-state \
|
|
48
|
+
--source-owner-uid 1001 \
|
|
49
|
+
--lock-owner-uid 1001 \
|
|
50
|
+
--snapshot-owner-uid 1002 \
|
|
51
|
+
--database-artifact database.duckdb \
|
|
52
|
+
--wal-artifact app.duckdb.wal \
|
|
53
|
+
--metadata-json '{"application":"example"}' \
|
|
54
|
+
--release release-42 \
|
|
55
|
+
snapshot --keep 7
|
|
56
|
+
|
|
57
|
+
duckdb-safe-snapshot \
|
|
58
|
+
--database /var/lib/example/app.duckdb --wal /var/lib/example/app.duckdb.wal \
|
|
59
|
+
--lock /run/example/writer.lock \
|
|
60
|
+
--backup-root /var/backups/example/snapshots \
|
|
61
|
+
--state-root /var/lib/example/snapshot-state \
|
|
62
|
+
--source-owner-uid 1001 --lock-owner-uid 1001 --snapshot-owner-uid 1002 \
|
|
63
|
+
--database-artifact database.duckdb --wal-artifact app.duckdb.wal \
|
|
64
|
+
verify latest
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The JSON response includes the timestamped snapshot id, artifact SHA-256 values,
|
|
68
|
+
and pruned recognized snapshots. Copy the database artifact and, when present,
|
|
69
|
+
the WAL artifact to the same recovery location before opening the recovery copy.
|
|
70
|
+
|
|
71
|
+
## Safety model
|
|
72
|
+
|
|
73
|
+
This package does not make an uncoordinated DuckDB writer safe. Every writer
|
|
74
|
+
that can modify the database or WAL must acquire the exact same Linux `flock`
|
|
75
|
+
lock before changing either file. The snapshot takes that exclusive lock, checks
|
|
76
|
+
source identity before and after copying, and fails if it observes a change.
|
|
77
|
+
|
|
78
|
+
Completed snapshot directories are created privately (`0700`) and contain only
|
|
79
|
+
the configured database artifact, optional WAL, a canonical JSON manifest, and
|
|
80
|
+
its SHA-256 checksum (`0600`). A temporary directory becomes visible as a
|
|
81
|
+
completed set only through `rename`; the set is then verified before `latest`
|
|
82
|
+
is atomically updated. Verification rejects symlinks, hardlinked files,
|
|
83
|
+
unexpected ownership or modes, missing/extra entries, malformed manifests, and
|
|
84
|
+
checksum or size mismatches. Retention deletes only recognized, already-verified
|
|
85
|
+
timestamped sets.
|
|
86
|
+
|
|
87
|
+
`Config` resolves existing path aliases and rejects relative paths, output-root
|
|
88
|
+
aliases, output-root nesting, and
|
|
89
|
+
database/lock paths inside private output roots. It also requires all three
|
|
90
|
+
owner UIDs and simple, distinct database/WAL artifact names. The caller must
|
|
91
|
+
create the shared lock under an ownership/mode policy accepted by `Config`.
|
|
92
|
+
For a service launcher that acquires this lock before selecting application
|
|
93
|
+
code, `inherited_lock_fd` may carry that open descriptor into `create_snapshot`.
|
|
94
|
+
It must be a non-negative integer descriptor for the exact configured lock;
|
|
95
|
+
the package validates its inode and reuses it without closing or unlocking it,
|
|
96
|
+
so the launcher retains the lock for the full child-process lifetime.
|
|
97
|
+
|
|
98
|
+
This is intentionally not a restore tool, service manager, or scheduler. Its
|
|
99
|
+
file checks protect leaf files and configured private roots; callers must also
|
|
100
|
+
keep each parent directory trusted against replacement. Use a
|
|
101
|
+
disposable recovery copy to test your own restoration procedure. The package
|
|
102
|
+
uses POSIX `fcntl.flock`, `O_NOFOLLOW`, directory fsync, and POSIX permissions;
|
|
103
|
+
it is supported on Linux filesystems with those semantics. Network filesystems
|
|
104
|
+
and writers that do not honor the shared lock are outside its consistency claim.
|
|
105
|
+
|
|
106
|
+
## Python API
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from pathlib import Path
|
|
110
|
+
from duckdb_safe_snapshot import Config, create_snapshot, verify_latest
|
|
111
|
+
|
|
112
|
+
config = Config(
|
|
113
|
+
database_path=Path("/var/lib/example/app.duckdb"),
|
|
114
|
+
wal_path=Path("/var/lib/example/app.duckdb.wal"),
|
|
115
|
+
lock_path=Path("/run/example/writer.lock"),
|
|
116
|
+
backup_root=Path("/var/backups/example/snapshots"),
|
|
117
|
+
state_root=Path("/var/lib/example/snapshot-state"),
|
|
118
|
+
source_owner_uid=1001,
|
|
119
|
+
lock_owner_uid=1001,
|
|
120
|
+
snapshot_owner_uid=1002,
|
|
121
|
+
database_artifact_name="database.duckdb",
|
|
122
|
+
wal_artifact_name="app.duckdb.wal",
|
|
123
|
+
metadata={"application": "example"},
|
|
124
|
+
release=lambda: "release-42", # optional caller-owned identifier
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
created = create_snapshot(config, keep=7)
|
|
128
|
+
assert verify_latest(config)["snapshot_id"] == created["snapshot_id"]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Compatibility and maintenance
|
|
132
|
+
|
|
133
|
+
The source requires Python 3.10–3.12. CI tests those Python versions with
|
|
134
|
+
DuckDB 1.5.5, the version used to produce the initial real database/WAL recovery
|
|
135
|
+
test. DuckDB is not a runtime dependency because this package copies files; it
|
|
136
|
+
is a test dependency only. No compatibility promise is made for other DuckDB,
|
|
137
|
+
Python, operating-system, or filesystem versions until they are tested.
|
|
138
|
+
|
|
139
|
+
An application can pass `release` as a string or no-argument callback. Its
|
|
140
|
+
result (a string or `null`) is recorded as the manifest's top-level `release`
|
|
141
|
+
field while the writer lock is held. Release meaning and validation stay with
|
|
142
|
+
the application; existing schema-1 manifests without that field remain
|
|
143
|
+
verifiable.
|
|
144
|
+
|
|
145
|
+
This project was extracted from a private application maintained by
|
|
146
|
+
[Sid Kalla](https://github.com/sidko). The public package contains synthetic
|
|
147
|
+
tests and generic configuration only. It is Apache-2.0 licensed; that license
|
|
148
|
+
does not grant rights to Gale Finance names, logos, or visual identity. Support
|
|
149
|
+
is best effort from the current release.
|
|
150
|
+
|
|
151
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md), [SECURITY.md](SECURITY.md), and
|
|
152
|
+
[AGENT_INTEGRATION.md](AGENT_INTEGRATION.md).
|
|
153
|
+
|
|
154
|
+
The public history begins with the 2026 extraction work. The private source
|
|
155
|
+
mapping is retained outside this repository because it names Gale-only paths
|
|
156
|
+
and operational context. Some early development used Claude as a coding
|
|
157
|
+
assistant; Sid Kalla selected, reviewed and maintains this code.
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# duckdb-safe-snapshot
|
|
2
|
+
|
|
3
|
+
Create private, checksummed snapshot sets of a DuckDB database and its WAL while
|
|
4
|
+
cooperating writers share one external file lock. It is a small Linux library
|
|
5
|
+
and CLI for applications that own their own scheduling and recovery process.
|
|
6
|
+
|
|
7
|
+

|
|
8
|
+
|
|
9
|
+
The image is an illustrative static terminal transcript with synthetic values;
|
|
10
|
+
it is not captured output from a snapshot run.
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
python -m pip install duckdb-safe-snapshot==0.1.0
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Every path and owner policy is explicit; the package has no application,
|
|
17
|
+
account, service, or production-path defaults.
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
duckdb-safe-snapshot \
|
|
21
|
+
--database /var/lib/example/app.duckdb \
|
|
22
|
+
--wal /var/lib/example/app.duckdb.wal \
|
|
23
|
+
--lock /run/example/writer.lock \
|
|
24
|
+
--backup-root /var/backups/example/snapshots \
|
|
25
|
+
--state-root /var/lib/example/snapshot-state \
|
|
26
|
+
--source-owner-uid 1001 \
|
|
27
|
+
--lock-owner-uid 1001 \
|
|
28
|
+
--snapshot-owner-uid 1002 \
|
|
29
|
+
--database-artifact database.duckdb \
|
|
30
|
+
--wal-artifact app.duckdb.wal \
|
|
31
|
+
--metadata-json '{"application":"example"}' \
|
|
32
|
+
--release release-42 \
|
|
33
|
+
snapshot --keep 7
|
|
34
|
+
|
|
35
|
+
duckdb-safe-snapshot \
|
|
36
|
+
--database /var/lib/example/app.duckdb --wal /var/lib/example/app.duckdb.wal \
|
|
37
|
+
--lock /run/example/writer.lock \
|
|
38
|
+
--backup-root /var/backups/example/snapshots \
|
|
39
|
+
--state-root /var/lib/example/snapshot-state \
|
|
40
|
+
--source-owner-uid 1001 --lock-owner-uid 1001 --snapshot-owner-uid 1002 \
|
|
41
|
+
--database-artifact database.duckdb --wal-artifact app.duckdb.wal \
|
|
42
|
+
verify latest
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The JSON response includes the timestamped snapshot id, artifact SHA-256 values,
|
|
46
|
+
and pruned recognized snapshots. Copy the database artifact and, when present,
|
|
47
|
+
the WAL artifact to the same recovery location before opening the recovery copy.
|
|
48
|
+
|
|
49
|
+
## Safety model
|
|
50
|
+
|
|
51
|
+
This package does not make an uncoordinated DuckDB writer safe. Every writer
|
|
52
|
+
that can modify the database or WAL must acquire the exact same Linux `flock`
|
|
53
|
+
lock before changing either file. The snapshot takes that exclusive lock, checks
|
|
54
|
+
source identity before and after copying, and fails if it observes a change.
|
|
55
|
+
|
|
56
|
+
Completed snapshot directories are created privately (`0700`) and contain only
|
|
57
|
+
the configured database artifact, optional WAL, a canonical JSON manifest, and
|
|
58
|
+
its SHA-256 checksum (`0600`). A temporary directory becomes visible as a
|
|
59
|
+
completed set only through `rename`; the set is then verified before `latest`
|
|
60
|
+
is atomically updated. Verification rejects symlinks, hardlinked files,
|
|
61
|
+
unexpected ownership or modes, missing/extra entries, malformed manifests, and
|
|
62
|
+
checksum or size mismatches. Retention deletes only recognized, already-verified
|
|
63
|
+
timestamped sets.
|
|
64
|
+
|
|
65
|
+
`Config` resolves existing path aliases and rejects relative paths, output-root
|
|
66
|
+
aliases, output-root nesting, and
|
|
67
|
+
database/lock paths inside private output roots. It also requires all three
|
|
68
|
+
owner UIDs and simple, distinct database/WAL artifact names. The caller must
|
|
69
|
+
create the shared lock under an ownership/mode policy accepted by `Config`.
|
|
70
|
+
For a service launcher that acquires this lock before selecting application
|
|
71
|
+
code, `inherited_lock_fd` may carry that open descriptor into `create_snapshot`.
|
|
72
|
+
It must be a non-negative integer descriptor for the exact configured lock;
|
|
73
|
+
the package validates its inode and reuses it without closing or unlocking it,
|
|
74
|
+
so the launcher retains the lock for the full child-process lifetime.
|
|
75
|
+
|
|
76
|
+
This is intentionally not a restore tool, service manager, or scheduler. Its
|
|
77
|
+
file checks protect leaf files and configured private roots; callers must also
|
|
78
|
+
keep each parent directory trusted against replacement. Use a
|
|
79
|
+
disposable recovery copy to test your own restoration procedure. The package
|
|
80
|
+
uses POSIX `fcntl.flock`, `O_NOFOLLOW`, directory fsync, and POSIX permissions;
|
|
81
|
+
it is supported on Linux filesystems with those semantics. Network filesystems
|
|
82
|
+
and writers that do not honor the shared lock are outside its consistency claim.
|
|
83
|
+
|
|
84
|
+
## Python API
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from pathlib import Path
|
|
88
|
+
from duckdb_safe_snapshot import Config, create_snapshot, verify_latest
|
|
89
|
+
|
|
90
|
+
config = Config(
|
|
91
|
+
database_path=Path("/var/lib/example/app.duckdb"),
|
|
92
|
+
wal_path=Path("/var/lib/example/app.duckdb.wal"),
|
|
93
|
+
lock_path=Path("/run/example/writer.lock"),
|
|
94
|
+
backup_root=Path("/var/backups/example/snapshots"),
|
|
95
|
+
state_root=Path("/var/lib/example/snapshot-state"),
|
|
96
|
+
source_owner_uid=1001,
|
|
97
|
+
lock_owner_uid=1001,
|
|
98
|
+
snapshot_owner_uid=1002,
|
|
99
|
+
database_artifact_name="database.duckdb",
|
|
100
|
+
wal_artifact_name="app.duckdb.wal",
|
|
101
|
+
metadata={"application": "example"},
|
|
102
|
+
release=lambda: "release-42", # optional caller-owned identifier
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
created = create_snapshot(config, keep=7)
|
|
106
|
+
assert verify_latest(config)["snapshot_id"] == created["snapshot_id"]
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Compatibility and maintenance
|
|
110
|
+
|
|
111
|
+
The source requires Python 3.10–3.12. CI tests those Python versions with
|
|
112
|
+
DuckDB 1.5.5, the version used to produce the initial real database/WAL recovery
|
|
113
|
+
test. DuckDB is not a runtime dependency because this package copies files; it
|
|
114
|
+
is a test dependency only. No compatibility promise is made for other DuckDB,
|
|
115
|
+
Python, operating-system, or filesystem versions until they are tested.
|
|
116
|
+
|
|
117
|
+
An application can pass `release` as a string or no-argument callback. Its
|
|
118
|
+
result (a string or `null`) is recorded as the manifest's top-level `release`
|
|
119
|
+
field while the writer lock is held. Release meaning and validation stay with
|
|
120
|
+
the application; existing schema-1 manifests without that field remain
|
|
121
|
+
verifiable.
|
|
122
|
+
|
|
123
|
+
This project was extracted from a private application maintained by
|
|
124
|
+
[Sid Kalla](https://github.com/sidko). The public package contains synthetic
|
|
125
|
+
tests and generic configuration only. It is Apache-2.0 licensed; that license
|
|
126
|
+
does not grant rights to Gale Finance names, logos, or visual identity. Support
|
|
127
|
+
is best effort from the current release.
|
|
128
|
+
|
|
129
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md), [SECURITY.md](SECURITY.md), and
|
|
130
|
+
[AGENT_INTEGRATION.md](AGENT_INTEGRATION.md).
|
|
131
|
+
|
|
132
|
+
The public history begins with the 2026 extraction work. The private source
|
|
133
|
+
mapping is retained outside this repository because it names Gale-only paths
|
|
134
|
+
and operational context. Some early development used Claude as a coding
|
|
135
|
+
assistant; Sid Kalla selected, reviewed and maintains this code.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Releasing
|
|
2
|
+
|
|
3
|
+
After `main` CI passes, push the matching `v<version>` tag over SSH. The release
|
|
4
|
+
workflow checks out the event SHA, validates the version, builds the package,
|
|
5
|
+
and runs the clean-consumer test. PyPI retries compare normalized archive
|
|
6
|
+
contents, ignore container timestamps, and skip only matching files. A GitHub
|
|
7
|
+
Release follows a successful registry publish.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Security policy
|
|
2
|
+
|
|
3
|
+
Supported versions: the latest `0.x` release.
|
|
4
|
+
|
|
5
|
+
Please use GitHub's private vulnerability reporting for this repository. Do not
|
|
6
|
+
include database files, credentials, customer paths, or recovery artifacts in a
|
|
7
|
+
public issue. If private reporting is unavailable, open a minimal public issue
|
|
8
|
+
asking for a secure contact channel.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="840" height="230" viewBox="0 0 840 230" role="img" aria-label="Illustrative terminal transcript with synthetic DuckDB snapshot values">
|
|
2
|
+
<rect width="840" height="230" rx="14" fill="#101722"/>
|
|
3
|
+
<circle cx="34" cy="30" r="8" fill="#ff6b6b"/><circle cx="58" cy="30" r="8" fill="#ffd166"/><circle cx="82" cy="30" r="8" fill="#06d6a0"/>
|
|
4
|
+
<g font-family="ui-monospace, SFMono-Regular, Menlo, monospace" font-size="17">
|
|
5
|
+
<text x="28" y="58" fill="#a9bfd7">Illustrative terminal transcript — synthetic values, not a captured run</text>
|
|
6
|
+
<text x="28" y="92" fill="#9cdcfe">$ duckdb-safe-snapshot … snapshot --keep 7</text>
|
|
7
|
+
<text x="28" y="126" fill="#b5cea8">{"artifacts":[{"name":"database.duckdb","size":53248}],</text>
|
|
8
|
+
<text x="28" y="155" fill="#b5cea8"> "snapshot_id":"20260909T010203.000000Z","status":"created"}</text>
|
|
9
|
+
<text x="28" y="202" fill="#9cdcfe">$ duckdb-safe-snapshot … verify latest</text>
|
|
10
|
+
<text x="585" y="202" fill="#4ec9b0">"status":"verified"</text>
|
|
11
|
+
</g>
|
|
12
|
+
</svg>
|