sqlite-sync-core 0.2.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlite_sync_core-0.5.0/PKG-INFO +176 -0
- sqlite_sync_core-0.5.0/README.md +138 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/pyproject.toml +4 -1
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/__init__.py +1 -1
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/bundle/generate.py +5 -2
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/db/connection.py +41 -6
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/db/migrations.py +4 -1
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/db/schema.py +15 -56
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/db/triggers.py +15 -6
- sqlite_sync_core-0.5.0/src/sqlite_sync/engine.py +214 -0
- sqlite_sync_core-0.5.0/src/sqlite_sync/ext/cli/main.py +110 -0
- sqlite_sync_core-0.5.0/src/sqlite_sync/ext/network_manager.py +117 -0
- sqlite_sync_core-0.5.0/src/sqlite_sync/ext/node.py +112 -0
- {sqlite_sync_core-0.2.0/src/sqlite_sync → sqlite_sync_core-0.5.0/src/sqlite_sync/ext}/sync_loop.py +9 -0
- sqlite_sync_core-0.5.0/src/sqlite_sync/hlc.py +95 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/import_apply/apply.py +74 -28
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/import_apply/conflict.py +7 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/log/operations.py +14 -44
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/log/vector_clock.py +25 -44
- sqlite_sync_core-0.5.0/src/sqlite_sync/resolution/lww_merge.py +87 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/schema_evolution.py +9 -1
- sqlite_sync_core-0.5.0/src/sqlite_sync_core.egg-info/PKG-INFO +176 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync_core.egg-info/SOURCES.txt +10 -4
- sqlite_sync_core-0.5.0/src/sqlite_sync_core.egg-info/entry_points.txt +2 -0
- sqlite_sync_core-0.2.0/PKG-INFO +0 -379
- sqlite_sync_core-0.2.0/README.md +0 -341
- sqlite_sync_core-0.2.0/src/sqlite_sync/engine.py +0 -541
- sqlite_sync_core-0.2.0/src/sqlite_sync_core.egg-info/PKG-INFO +0 -379
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/LICENSE +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/setup.cfg +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/audit/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/audit/import_log.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/bundle/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/bundle/format.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/bundle/validate.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/capture/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/capture/change_capture.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/config.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/crash_safety.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/db/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/errors.py +0 -0
- {sqlite_sync_core-0.2.0/src/sqlite_sync → sqlite_sync_core-0.5.0/src/sqlite_sync/ext}/server/__init__.py +0 -0
- {sqlite_sync_core-0.2.0/src/sqlite_sync → sqlite_sync_core-0.5.0/src/sqlite_sync/ext}/server/auth.py +0 -0
- {sqlite_sync_core-0.2.0/src/sqlite_sync → sqlite_sync_core-0.5.0/src/sqlite_sync/ext}/server/http_server.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/import_apply/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/import_apply/dedup.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/import_apply/ordering.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/invariants.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/log/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/log_compaction.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/metrics.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/network/client.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/network/peer_discovery.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/network/protocol.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/network/server.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/resolution/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/security.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/transport/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/transport/base.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/transport/http_transport.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/transport/websocket_transport.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/utils/__init__.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/utils/hashing.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/utils/msgpack_codec.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync/utils/uuid7.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync_core.egg-info/dependency_links.txt +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync_core.egg-info/requires.txt +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/src/sqlite_sync_core.egg-info/top_level.txt +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/tests/test_conflicts.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/tests/test_determinism.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/tests/test_idempotency.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/tests/test_invariants.py +0 -0
- {sqlite_sync_core-0.2.0 → sqlite_sync_core-0.5.0}/tests/test_minimal.py +0 -0
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sqlite-sync-core
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Universal SQLite Synchronization Core - A dependency-grade, local-first, offline-first SQLite synchronization primitive
|
|
5
|
+
License: AGPL-3.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/shivay00001/sqlite-sync-core
|
|
7
|
+
Project-URL: Documentation, https://github.com/shivay00001/sqlite-sync-core#readme
|
|
8
|
+
Project-URL: Repository, https://github.com/shivay00001/sqlite-sync-core.git
|
|
9
|
+
Project-URL: Issues, https://github.com/shivay00001/sqlite-sync-core/issues
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Database
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: msgpack>=1.0.0
|
|
24
|
+
Requires-Dist: websockets>=12.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
29
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
30
|
+
Provides-Extra: server
|
|
31
|
+
Requires-Dist: flask>=3.0.0; extra == "server"
|
|
32
|
+
Provides-Extra: crypto
|
|
33
|
+
Requires-Dist: cryptography>=41.0.0; extra == "crypto"
|
|
34
|
+
Provides-Extra: all
|
|
35
|
+
Requires-Dist: flask>=3.0.0; extra == "all"
|
|
36
|
+
Requires-Dist: cryptography>=41.0.0; extra == "all"
|
|
37
|
+
Dynamic: license-file
|
|
38
|
+
|
|
39
|
+
# sqlite-sync-core
|
|
40
|
+
|
|
41
|
+
[](https://www.python.org/downloads/)
|
|
42
|
+
[](https://www.gnu.org/licenses/agpl-3.0)
|
|
43
|
+
[](https://pypi.org/project/sqlite-sync-core/)
|
|
44
|
+
[](https://github.com/shivay00001/sqlite-sync-core)
|
|
45
|
+
|
|
46
|
+
**A production-grade, turn-key synchronization system for SQLite.**
|
|
47
|
+
|
|
48
|
+
`sqlite-sync-core` provides a powerful, local-first synchronization engine that works seamlessly across multi-peer networks. It handles the "hard parts" of sync (vector clocks, causality, delta bundles, and conflict resolution) while providing a simple, turn-key interface for developers.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 🚀 Turn-Key Synchronization
|
|
53
|
+
|
|
54
|
+
You can launch a full synchronization node in one command. No infrastructure required.
|
|
55
|
+
|
|
56
|
+
### 16-Second Setup (CLI)
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# Install the package
|
|
60
|
+
pip install sqlite-sync-core
|
|
61
|
+
|
|
62
|
+
# Start a node and sync the 'tasks' table automatically
|
|
63
|
+
sqlite-sync start --db app.db --name Device-A --tables tasks
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### Automatic Multi-Peer Sync
|
|
67
|
+
|
|
68
|
+
Nodes automatically discover each other on the local network (P2P) and synchronize state in the background without any manual peer configuration.
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## 🏗️ Enterprise Features
|
|
73
|
+
|
|
74
|
+
- **Multi-Peer Orchestration**: Automatically scales sync across N devices.
|
|
75
|
+
- **P2P Discovery**: Zero-config peer-to-peer discovery on LAN.
|
|
76
|
+
- **Automatic Resolution**: Configurable strategies like Last-Write-Wins and Field-Level Merge.
|
|
77
|
+
- **Schema Evolution**: Built-in migrations that sync across the network.
|
|
78
|
+
- **Transport Agnostic**: Works over HTTP, WebSockets, or file transfer.
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Technical Usage (Library)
|
|
83
|
+
|
|
84
|
+
### Initialize a Node in Code
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from sqlite_sync import SyncNode
|
|
88
|
+
|
|
89
|
+
node = SyncNode(
|
|
90
|
+
db_path="app.db",
|
|
91
|
+
device_name="MobileApp",
|
|
92
|
+
sync_interval=10 # Sync every 10 seconds
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
await node.start()
|
|
96
|
+
node.enable_sync_for_table("users")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### Safe Schema Migrations
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
# Safely add a column that will sync to all other peers
|
|
103
|
+
sqlite-sync migrate --db app.db --table tasks --add-column priority --type INTEGER
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## Core Invariants
|
|
109
|
+
|
|
110
|
+
| # | Invariant | Description |
|
|
111
|
+
|---|-----------|-------------|
|
|
112
|
+
| 1 | **Causal consistency** | Vector clocks ensure the correct order of operations. |
|
|
113
|
+
| 2 | **Deterministic Replay** | Identical sets of operations always result in identical state. |
|
|
114
|
+
| 3 | **Conflict Tolerance** | Detects and resolves conflicts explicitly and safely. |
|
|
115
|
+
| 4 | **Offline-First** | Entirely local-first design; works without cloud or internet. |
|
|
116
|
+
|
|
117
|
+
### 2. Generate a Delta Bundle
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
# To be sent to Peer B
|
|
121
|
+
bundle_path = engine.generate_bundle(
|
|
122
|
+
peer_device_id=peer_b_id,
|
|
123
|
+
output_path="delta.db"
|
|
124
|
+
)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### 3. Import and Detect Conflicts
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
# On Peer B
|
|
131
|
+
result = engine.import_bundle("delta.db")
|
|
132
|
+
|
|
133
|
+
print(f"Ops Applied: {result.applied_count}")
|
|
134
|
+
print(f"Conflicts Detected: {result.conflict_count}")
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## Core Invariants
|
|
140
|
+
|
|
141
|
+
| # | Invariant | Description |
|
|
142
|
+
|---|-----------|-------------|
|
|
143
|
+
| 1 | **Append-only** | Operation history is immutable. |
|
|
144
|
+
| 2 | **Causal consistency** | Hybrid Logical Clocks (HLC) ensure correct partial ordering and wall-clock correlation. |
|
|
145
|
+
| 3 | **Deterministic** | Replay results are identical across all replicas. |
|
|
146
|
+
| 4 | **Field-Level Merge** | "Smart" LWW resolution merges concurrent non-conflicting field updates. |
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## Architecture
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
┌─────────────────────────────────┐
|
|
154
|
+
│ Your Sync System / App │
|
|
155
|
+
└───────────────┬─────────────────┘
|
|
156
|
+
│ Uses
|
|
157
|
+
┌───────────────▼─────────────────┐
|
|
158
|
+
│ sqlite-sync-core │
|
|
159
|
+
│ (Logging, Bundling, Clocks) │
|
|
160
|
+
└───────────────┬─────────────────┘
|
|
161
|
+
│ Persists to
|
|
162
|
+
┌───────────────▼─────────────────┐
|
|
163
|
+
│ SQLite Database │
|
|
164
|
+
└─────────────────────────────────┘
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
**AGPL-3.0** for Open Source.
|
|
172
|
+
Contact <shivaysinghrajput@proton.me> for commercial licensing.
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
**Built for developers who need a reliable sync foundation.**
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# sqlite-sync-core
|
|
2
|
+
|
|
3
|
+
[](https://www.python.org/downloads/)
|
|
4
|
+
[](https://www.gnu.org/licenses/agpl-3.0)
|
|
5
|
+
[](https://pypi.org/project/sqlite-sync-core/)
|
|
6
|
+
[](https://github.com/shivay00001/sqlite-sync-core)
|
|
7
|
+
|
|
8
|
+
**A production-grade, turn-key synchronization system for SQLite.**
|
|
9
|
+
|
|
10
|
+
`sqlite-sync-core` provides a powerful, local-first synchronization engine that works seamlessly across multi-peer networks. It handles the "hard parts" of sync (vector clocks, causality, delta bundles, and conflict resolution) while providing a simple, turn-key interface for developers.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## 🚀 Turn-Key Synchronization
|
|
15
|
+
|
|
16
|
+
You can launch a full synchronization node in one command. No infrastructure required.
|
|
17
|
+
|
|
18
|
+
### 16-Second Setup (CLI)
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
# Install the package
|
|
22
|
+
pip install sqlite-sync-core
|
|
23
|
+
|
|
24
|
+
# Start a node and sync the 'tasks' table automatically
|
|
25
|
+
sqlite-sync start --db app.db --name Device-A --tables tasks
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Automatic Multi-Peer Sync
|
|
29
|
+
|
|
30
|
+
Nodes automatically discover each other on the local network (P2P) and synchronize state in the background without any manual peer configuration.
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## 🏗️ Enterprise Features
|
|
35
|
+
|
|
36
|
+
- **Multi-Peer Orchestration**: Automatically scales sync across N devices.
|
|
37
|
+
- **P2P Discovery**: Zero-config peer-to-peer discovery on LAN.
|
|
38
|
+
- **Automatic Resolution**: Configurable strategies like Last-Write-Wins and Field-Level Merge.
|
|
39
|
+
- **Schema Evolution**: Built-in migrations that sync across the network.
|
|
40
|
+
- **Transport Agnostic**: Works over HTTP, WebSockets, or file transfer.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Technical Usage (Library)
|
|
45
|
+
|
|
46
|
+
### Initialize a Node in Code
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from sqlite_sync import SyncNode
|
|
50
|
+
|
|
51
|
+
node = SyncNode(
|
|
52
|
+
db_path="app.db",
|
|
53
|
+
device_name="MobileApp",
|
|
54
|
+
sync_interval=10 # Sync every 10 seconds
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
await node.start()
|
|
58
|
+
node.enable_sync_for_table("users")
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Safe Schema Migrations
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# Safely add a column that will sync to all other peers
|
|
65
|
+
sqlite-sync migrate --db app.db --table tasks --add-column priority --type INTEGER
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Core Invariants
|
|
71
|
+
|
|
72
|
+
| # | Invariant | Description |
|
|
73
|
+
|---|-----------|-------------|
|
|
74
|
+
| 1 | **Causal consistency** | Vector clocks ensure the correct order of operations. |
|
|
75
|
+
| 2 | **Deterministic Replay** | Identical sets of operations always result in identical state. |
|
|
76
|
+
| 3 | **Conflict Tolerance** | Detects and resolves conflicts explicitly and safely. |
|
|
77
|
+
| 4 | **Offline-First** | Entirely local-first design; works without cloud or internet. |
|
|
78
|
+
|
|
79
|
+
### 2. Generate a Delta Bundle
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
# To be sent to Peer B
|
|
83
|
+
bundle_path = engine.generate_bundle(
|
|
84
|
+
peer_device_id=peer_b_id,
|
|
85
|
+
output_path="delta.db"
|
|
86
|
+
)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### 3. Import and Detect Conflicts
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
# On Peer B
|
|
93
|
+
result = engine.import_bundle("delta.db")
|
|
94
|
+
|
|
95
|
+
print(f"Ops Applied: {result.applied_count}")
|
|
96
|
+
print(f"Conflicts Detected: {result.conflict_count}")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Core Invariants
|
|
102
|
+
|
|
103
|
+
| # | Invariant | Description |
|
|
104
|
+
|---|-----------|-------------|
|
|
105
|
+
| 1 | **Append-only** | Operation history is immutable. |
|
|
106
|
+
| 2 | **Causal consistency** | Hybrid Logical Clocks (HLC) ensure correct partial ordering and wall-clock correlation. |
|
|
107
|
+
| 3 | **Deterministic** | Replay results are identical across all replicas. |
|
|
108
|
+
| 4 | **Field-Level Merge** | "Smart" LWW resolution merges concurrent non-conflicting field updates. |
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## Architecture
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
┌─────────────────────────────────┐
|
|
116
|
+
│ Your Sync System / App │
|
|
117
|
+
└───────────────┬─────────────────┘
|
|
118
|
+
│ Uses
|
|
119
|
+
┌───────────────▼─────────────────┐
|
|
120
|
+
│ sqlite-sync-core │
|
|
121
|
+
│ (Logging, Bundling, Clocks) │
|
|
122
|
+
└───────────────┬─────────────────┘
|
|
123
|
+
│ Persists to
|
|
124
|
+
┌───────────────▼─────────────────┐
|
|
125
|
+
│ SQLite Database │
|
|
126
|
+
└─────────────────────────────────┘
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
**AGPL-3.0** for Open Source.
|
|
134
|
+
Contact <shivaysinghrajput@proton.me> for commercial licensing.
|
|
135
|
+
|
|
136
|
+
---
|
|
137
|
+
|
|
138
|
+
**Built for developers who need a reliable sync foundation.**
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "sqlite-sync-core"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.5.0"
|
|
8
8
|
description = "Universal SQLite Synchronization Core - A dependency-grade, local-first, offline-first SQLite synchronization primitive"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "AGPL-3.0"}
|
|
@@ -50,6 +50,9 @@ all = [
|
|
|
50
50
|
"cryptography>=41.0.0",
|
|
51
51
|
]
|
|
52
52
|
|
|
53
|
+
[project.scripts]
|
|
54
|
+
sqlite-sync = "sqlite_sync.cli.main:main"
|
|
55
|
+
|
|
53
56
|
[tool.setuptools.packages.find]
|
|
54
57
|
where = ["src"]
|
|
55
58
|
|
|
@@ -5,6 +5,7 @@ Generates a bundle containing operations that a peer hasn't seen.
|
|
|
5
5
|
Bundles are self-contained SQLite files.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
import os
|
|
8
9
|
import sqlite3
|
|
9
10
|
import time
|
|
10
11
|
from typing import Final
|
|
@@ -56,6 +57,8 @@ def generate_bundle(
|
|
|
56
57
|
|
|
57
58
|
# Create bundle database
|
|
58
59
|
try:
|
|
60
|
+
if os.path.exists(output_path):
|
|
61
|
+
os.remove(output_path)
|
|
59
62
|
bundle_conn = sqlite3.connect(output_path)
|
|
60
63
|
|
|
61
64
|
# Create bundle schema
|
|
@@ -66,10 +69,10 @@ def generate_bundle(
|
|
|
66
69
|
bundle_conn.executemany(
|
|
67
70
|
"""
|
|
68
71
|
INSERT INTO bundle_operations (
|
|
69
|
-
op_id, device_id, parent_op_id, vector_clock,
|
|
72
|
+
op_id, device_id, parent_op_id, vector_clock, hlc,
|
|
70
73
|
table_name, op_type, row_pk, old_values, new_values,
|
|
71
74
|
schema_version, created_at, is_local, applied_at
|
|
72
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
75
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
73
76
|
""",
|
|
74
77
|
operations,
|
|
75
78
|
)
|
|
@@ -7,9 +7,14 @@ registration of application-defined SQL functions.
|
|
|
7
7
|
All connections use WAL mode for concurrent read/write.
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
+
import logging
|
|
10
11
|
import sqlite3
|
|
11
|
-
|
|
12
|
+
import json
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger("sqlite_sync.db")
|
|
12
16
|
|
|
17
|
+
from typing import Callable, Any
|
|
13
18
|
from sqlite_sync.config import SQLITE_PRAGMAS
|
|
14
19
|
from sqlite_sync.errors import DatabaseError
|
|
15
20
|
|
|
@@ -85,32 +90,62 @@ def _register_functions(conn: sqlite3.Connection) -> None:
|
|
|
85
90
|
from sqlite_sync.log.vector_clock import increment_vector_clock, merge_vector_clocks
|
|
86
91
|
from sqlite_sync.utils.msgpack_codec import pack_primary_key, pack_dict
|
|
87
92
|
import json
|
|
93
|
+
import sys
|
|
88
94
|
|
|
89
95
|
# sync_uuid_v7() -> BLOB(16)
|
|
90
|
-
|
|
96
|
+
def _uuid_v7() -> bytes:
|
|
97
|
+
return generate_uuid_v7()
|
|
98
|
+
|
|
99
|
+
conn.create_function("sync_uuid_v7", 0, _uuid_v7)
|
|
91
100
|
|
|
92
101
|
# sync_vector_clock_increment(device_id BLOB, vc_json TEXT) -> TEXT
|
|
93
|
-
def _vc_increment(device_id:
|
|
102
|
+
def _vc_increment(device_id: Any, vc_json: Any) -> str:
|
|
94
103
|
return increment_vector_clock(device_id, vc_json)
|
|
95
104
|
|
|
96
105
|
conn.create_function("sync_vector_clock_increment", 2, _vc_increment)
|
|
97
106
|
|
|
98
107
|
# sync_vector_clock_merge(vc1_json TEXT, vc2_json TEXT) -> TEXT
|
|
99
|
-
def _vc_merge(vc1_json:
|
|
108
|
+
def _vc_merge(vc1_json: Any, vc2_json: Any) -> str:
|
|
100
109
|
return merge_vector_clocks(vc1_json, vc2_json)
|
|
101
110
|
|
|
102
111
|
conn.create_function("sync_vector_clock_merge", 2, _vc_merge)
|
|
103
112
|
|
|
104
113
|
# sync_pack_pk(value) -> BLOB
|
|
105
|
-
|
|
114
|
+
def _pack_pk(value: Any) -> bytes:
|
|
115
|
+
return pack_primary_key(value)
|
|
116
|
+
|
|
117
|
+
conn.create_function("sync_pack_pk", 1, _pack_pk)
|
|
106
118
|
|
|
107
119
|
# sync_pack_values(json_str TEXT) -> BLOB
|
|
108
|
-
def _pack_values(json_str:
|
|
120
|
+
def _pack_values(json_str: Any) -> bytes:
|
|
121
|
+
if not json_str: return b""
|
|
109
122
|
data = json.loads(json_str)
|
|
110
123
|
return pack_dict(data)
|
|
111
124
|
|
|
112
125
|
conn.create_function("sync_pack_values", 1, _pack_values)
|
|
113
126
|
|
|
127
|
+
# sync_hlc_now(node_id TEXT) -> TEXT
|
|
128
|
+
def _hlc_now_placeholder(node_id: Any) -> str:
|
|
129
|
+
return f"{int(time.time() * 1000)}:0:{node_id}"
|
|
130
|
+
|
|
131
|
+
conn.create_function("sync_hlc_now", 1, _hlc_now_placeholder)
|
|
132
|
+
|
|
133
|
+
if not hasattr(_register_functions, "_disabled_state"):
|
|
134
|
+
_register_functions._disabled_state = {}
|
|
135
|
+
|
|
136
|
+
def _sync_is_disabled() -> int:
|
|
137
|
+
return _register_functions._disabled_state.get(id(conn), 0)
|
|
138
|
+
|
|
139
|
+
conn.create_function("sync_is_disabled", 0, _sync_is_disabled)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def set_sync_disabled(conn: sqlite3.Connection, disabled: bool) -> None:
|
|
143
|
+
"""Set the sync-disabled state for a specific connection."""
|
|
144
|
+
val = 1 if disabled else 0
|
|
145
|
+
if not hasattr(_register_functions, "_disabled_state"):
|
|
146
|
+
_register_functions._disabled_state = {}
|
|
147
|
+
_register_functions._disabled_state[id(conn)] = val
|
|
148
|
+
|
|
114
149
|
|
|
115
150
|
def execute_in_transaction(
|
|
116
151
|
conn: sqlite3.Connection,
|
|
@@ -49,10 +49,13 @@ def initialize_sync_tables(conn: sqlite3.Connection) -> bytes:
|
|
|
49
49
|
for sql in statement.strip().split(";"):
|
|
50
50
|
sql = sql.strip()
|
|
51
51
|
if sql:
|
|
52
|
+
# print(f"DEBUG: Executing SQL: {sql}")
|
|
52
53
|
conn.execute(sql)
|
|
53
54
|
except sqlite3.Error as e:
|
|
55
|
+
# Get the failing SQL if possible
|
|
56
|
+
failed_sql = locals().get('sql', 'unknown')
|
|
54
57
|
raise DatabaseError(
|
|
55
|
-
f"Failed to create sync tables: {e}",
|
|
58
|
+
f"Failed to create sync tables: {e} (SQL: {failed_sql})",
|
|
56
59
|
operation="create_tables",
|
|
57
60
|
) from e
|
|
58
61
|
|
|
@@ -1,61 +1,37 @@
|
|
|
1
1
|
"""
|
|
2
2
|
schema.py - Sync system table schema definitions.
|
|
3
|
-
|
|
4
|
-
Defines the exact schema for all sync tables per the specification.
|
|
5
|
-
All tables use STRICT mode for type enforcement.
|
|
6
3
|
"""
|
|
7
4
|
|
|
8
5
|
from typing import Final
|
|
9
6
|
|
|
10
|
-
# sync_operations table
|
|
11
|
-
# Every database mutation becomes a row here
|
|
7
|
+
# sync_operations table
|
|
12
8
|
SYNC_OPERATIONS_SCHEMA: Final[str] = """
|
|
13
9
|
CREATE TABLE IF NOT EXISTS sync_operations (
|
|
14
|
-
-- Identity (globally unique, time-ordered)
|
|
15
10
|
op_id BLOB PRIMARY KEY CHECK(length(op_id) = 16),
|
|
16
|
-
|
|
17
|
-
-- Source tracking
|
|
18
11
|
device_id BLOB NOT NULL CHECK(length(device_id) = 16),
|
|
19
|
-
|
|
20
|
-
-- Causality chain
|
|
21
12
|
parent_op_id BLOB CHECK(parent_op_id IS NULL OR length(parent_op_id) = 16),
|
|
22
13
|
vector_clock TEXT NOT NULL,
|
|
23
|
-
|
|
24
|
-
-- What changed
|
|
14
|
+
hlc TEXT NOT NULL,
|
|
25
15
|
table_name TEXT NOT NULL,
|
|
26
16
|
op_type TEXT NOT NULL CHECK(op_type IN ('INSERT', 'UPDATE', 'DELETE')),
|
|
27
17
|
row_pk BLOB NOT NULL,
|
|
28
|
-
|
|
29
|
-
-- Change content
|
|
30
18
|
old_values BLOB,
|
|
31
19
|
new_values BLOB,
|
|
32
|
-
|
|
33
|
-
-- Metadata
|
|
34
20
|
schema_version INTEGER NOT NULL,
|
|
35
21
|
created_at INTEGER NOT NULL,
|
|
36
|
-
|
|
37
|
-
-- Local tracking
|
|
38
22
|
is_local INTEGER NOT NULL CHECK(is_local IN (0, 1)),
|
|
39
23
|
applied_at INTEGER,
|
|
40
|
-
|
|
41
24
|
FOREIGN KEY (parent_op_id) REFERENCES sync_operations(op_id)
|
|
42
25
|
) STRICT;
|
|
43
26
|
"""
|
|
44
27
|
|
|
45
28
|
SYNC_OPERATIONS_INDICES: Final[str] = """
|
|
46
|
-
|
|
47
|
-
CREATE INDEX IF NOT EXISTS
|
|
48
|
-
ON sync_operations(device_id, created_at);
|
|
49
|
-
|
|
50
|
-
-- Index for conflict detection (find ops on same row)
|
|
51
|
-
CREATE INDEX IF NOT EXISTS idx_ops_table_pk
|
|
52
|
-
ON sync_operations(table_name, row_pk);
|
|
53
|
-
|
|
54
|
-
-- Index for deduplication
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_ops_device_created ON sync_operations(device_id, created_at);
|
|
30
|
+
CREATE INDEX IF NOT EXISTS idx_ops_table_pk ON sync_operations(table_name, row_pk);
|
|
55
31
|
CREATE INDEX IF NOT EXISTS idx_ops_id ON sync_operations(op_id);
|
|
56
32
|
"""
|
|
57
33
|
|
|
58
|
-
# sync_metadata table
|
|
34
|
+
# sync_metadata table
|
|
59
35
|
SYNC_METADATA_SCHEMA: Final[str] = """
|
|
60
36
|
CREATE TABLE IF NOT EXISTS sync_metadata (
|
|
61
37
|
key TEXT PRIMARY KEY,
|
|
@@ -63,24 +39,18 @@ CREATE TABLE IF NOT EXISTS sync_metadata (
|
|
|
63
39
|
) STRICT;
|
|
64
40
|
"""
|
|
65
41
|
|
|
66
|
-
# sync_conflicts table
|
|
42
|
+
# sync_conflicts table
|
|
67
43
|
SYNC_CONFLICTS_SCHEMA: Final[str] = """
|
|
68
44
|
CREATE TABLE IF NOT EXISTS sync_conflicts (
|
|
69
45
|
conflict_id BLOB PRIMARY KEY CHECK(length(conflict_id) = 16),
|
|
70
|
-
|
|
71
|
-
-- What conflicted
|
|
72
46
|
table_name TEXT NOT NULL,
|
|
73
47
|
row_pk BLOB NOT NULL,
|
|
74
|
-
|
|
75
|
-
-- Conflicting operations
|
|
76
48
|
local_op_id BLOB NOT NULL CHECK(length(local_op_id) = 16),
|
|
77
49
|
remote_op_id BLOB NOT NULL CHECK(length(remote_op_id) = 16),
|
|
78
|
-
|
|
79
|
-
-- Lifecycle
|
|
80
50
|
detected_at INTEGER NOT NULL,
|
|
81
51
|
resolved_at INTEGER,
|
|
82
52
|
resolution_op_id BLOB CHECK(resolution_op_id IS NULL OR length(resolution_op_id) = 16),
|
|
83
|
-
|
|
53
|
+
resolution_strategy TEXT,
|
|
84
54
|
FOREIGN KEY (local_op_id) REFERENCES sync_operations(op_id),
|
|
85
55
|
FOREIGN KEY (remote_op_id) REFERENCES sync_operations(op_id),
|
|
86
56
|
FOREIGN KEY (resolution_op_id) REFERENCES sync_operations(op_id)
|
|
@@ -88,45 +58,33 @@ CREATE TABLE IF NOT EXISTS sync_conflicts (
|
|
|
88
58
|
"""
|
|
89
59
|
|
|
90
60
|
SYNC_CONFLICTS_INDICES: Final[str] = """
|
|
91
|
-
CREATE INDEX IF NOT EXISTS idx_conflicts_unresolved
|
|
92
|
-
ON sync_conflicts(
|
|
93
|
-
|
|
94
|
-
CREATE INDEX IF NOT EXISTS idx_conflicts_row
|
|
95
|
-
ON sync_conflicts(table_name, row_pk);
|
|
61
|
+
CREATE INDEX IF NOT EXISTS idx_conflicts_unresolved ON sync_conflicts(detected_at) WHERE resolved_at IS NULL;
|
|
62
|
+
CREATE INDEX IF NOT EXISTS idx_conflicts_row ON sync_conflicts(table_name, row_pk);
|
|
96
63
|
"""
|
|
97
64
|
|
|
98
|
-
# sync_peer_state table
|
|
65
|
+
# sync_peer_state table
|
|
99
66
|
SYNC_PEER_STATE_SCHEMA: Final[str] = """
|
|
100
67
|
CREATE TABLE IF NOT EXISTS sync_peer_state (
|
|
101
68
|
peer_device_id BLOB PRIMARY KEY CHECK(length(peer_device_id) = 16),
|
|
102
|
-
|
|
103
|
-
-- What have we sent them?
|
|
104
69
|
last_sent_vector_clock TEXT NOT NULL,
|
|
105
70
|
last_sent_at INTEGER NOT NULL,
|
|
106
|
-
|
|
107
|
-
-- What have they sent us?
|
|
108
71
|
last_received_vector_clock TEXT NOT NULL,
|
|
109
72
|
last_received_at INTEGER NOT NULL
|
|
110
73
|
) STRICT;
|
|
111
74
|
"""
|
|
112
75
|
|
|
113
|
-
# sync_import_log table
|
|
76
|
+
# sync_import_log table
|
|
114
77
|
SYNC_IMPORT_LOG_SCHEMA: Final[str] = """
|
|
115
78
|
CREATE TABLE IF NOT EXISTS sync_import_log (
|
|
116
79
|
import_id BLOB PRIMARY KEY CHECK(length(import_id) = 16),
|
|
117
80
|
bundle_id BLOB NOT NULL CHECK(length(bundle_id) = 16),
|
|
118
81
|
bundle_hash BLOB NOT NULL CHECK(length(bundle_hash) = 32),
|
|
119
|
-
|
|
120
|
-
-- When and from whom
|
|
121
82
|
imported_at INTEGER NOT NULL,
|
|
122
83
|
source_device_id BLOB NOT NULL CHECK(length(source_device_id) = 16),
|
|
123
|
-
|
|
124
|
-
-- What happened
|
|
125
84
|
op_count INTEGER NOT NULL,
|
|
126
85
|
applied_count INTEGER NOT NULL,
|
|
127
86
|
conflict_count INTEGER NOT NULL,
|
|
128
87
|
duplicate_count INTEGER NOT NULL,
|
|
129
|
-
|
|
130
88
|
UNIQUE(bundle_hash)
|
|
131
89
|
) STRICT;
|
|
132
90
|
"""
|
|
@@ -147,9 +105,9 @@ ALL_SCHEMA_STATEMENTS: Final[tuple[str, ...]] = (
|
|
|
147
105
|
SYNC_IMPORT_LOG_INDICES,
|
|
148
106
|
)
|
|
149
107
|
|
|
150
|
-
# Bundle schema
|
|
108
|
+
# Bundle schema
|
|
151
109
|
BUNDLE_METADATA_SCHEMA: Final[str] = """
|
|
152
|
-
CREATE TABLE bundle_metadata (
|
|
110
|
+
CREATE TABLE IF NOT EXISTS bundle_metadata (
|
|
153
111
|
bundle_id BLOB PRIMARY KEY CHECK(length(bundle_id) = 16),
|
|
154
112
|
source_device_id BLOB NOT NULL CHECK(length(source_device_id) = 16),
|
|
155
113
|
created_at INTEGER NOT NULL,
|
|
@@ -160,11 +118,12 @@ CREATE TABLE bundle_metadata (
|
|
|
160
118
|
"""
|
|
161
119
|
|
|
162
120
|
BUNDLE_OPERATIONS_SCHEMA: Final[str] = """
|
|
163
|
-
CREATE TABLE bundle_operations (
|
|
121
|
+
CREATE TABLE IF NOT EXISTS bundle_operations (
|
|
164
122
|
op_id BLOB PRIMARY KEY CHECK(length(op_id) = 16),
|
|
165
123
|
device_id BLOB NOT NULL CHECK(length(device_id) = 16),
|
|
166
124
|
parent_op_id BLOB CHECK(parent_op_id IS NULL OR length(parent_op_id) = 16),
|
|
167
125
|
vector_clock TEXT NOT NULL,
|
|
126
|
+
hlc TEXT NOT NULL,
|
|
168
127
|
table_name TEXT NOT NULL,
|
|
169
128
|
op_type TEXT NOT NULL CHECK(op_type IN ('INSERT', 'UPDATE', 'DELETE')),
|
|
170
129
|
row_pk BLOB NOT NULL,
|