offline-sync-sdk 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- offline_sync_sdk-0.2.0.dist-info/METADATA +447 -0
- offline_sync_sdk-0.2.0.dist-info/RECORD +22 -0
- offline_sync_sdk-0.2.0.dist-info/WHEEL +4 -0
- offline_sync_sdk-0.2.0.dist-info/entry_points.txt +3 -0
- sync_sdk/__init__.py +39 -0
- sync_sdk/adapters/__init__.py +1 -0
- sync_sdk/adapters/direct.py +15 -0
- sync_sdk/adapters/http.py +71 -0
- sync_sdk/adapters/mongodb.py +473 -0
- sync_sdk/adapters/opentelemetry.py +231 -0
- sync_sdk/adapters/sqlalchemy.py +691 -0
- sync_sdk/adapters/sqlalchemy_config.py +145 -0
- sync_sdk/client.py +198 -0
- sync_sdk/gateway.py +163 -0
- sync_sdk/integrations/__init__.py +1 -0
- sync_sdk/integrations/fastapi.py +32 -0
- sync_sdk/models.py +41 -0
- sync_sdk/ports.py +94 -0
- sync_sdk/py.typed +1 -0
- sync_sdk/registry.py +44 -0
- sync_sdk/store.py +147 -0
- sync_sdk/telemetry.py +131 -0
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: offline-sync-sdk
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Embeddable synchronization pipelines with pluggable storage and transports
|
|
5
|
+
Keywords: offline-first,synchronization,sqlite,postgresql,mongodb
|
|
6
|
+
Author: VivekReddy10-08-2004
|
|
7
|
+
Author-email: VivekReddy10-08-2004 <vivek.bhimavarapu@maine.edu>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Framework :: AsyncIO
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Dist: pydantic>=2.11.0
|
|
19
|
+
Requires-Dist: fastapi>=0.116.0 ; extra == 'fastapi'
|
|
20
|
+
Requires-Dist: fastapi>=0.116.0 ; extra == 'gateway'
|
|
21
|
+
Requires-Dist: uvicorn[standard]>=0.35.0 ; extra == 'gateway'
|
|
22
|
+
Requires-Dist: pydantic-settings>=2.10.0 ; extra == 'gateway'
|
|
23
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.40 ; extra == 'gateway'
|
|
24
|
+
Requires-Dist: asyncpg>=0.30.0 ; extra == 'gateway'
|
|
25
|
+
Requires-Dist: alembic>=1.16.0 ; extra == 'gateway'
|
|
26
|
+
Requires-Dist: httpx>=0.28.0 ; extra == 'gateway'
|
|
27
|
+
Requires-Dist: psycopg[binary]>=3.3.5 ; extra == 'gateway'
|
|
28
|
+
Requires-Dist: asyncmy>=0.2.10 ; extra == 'gateway'
|
|
29
|
+
Requires-Dist: cryptography>=44.0.0 ; extra == 'gateway'
|
|
30
|
+
Requires-Dist: aiosqlite>=0.22.1 ; extra == 'gateway'
|
|
31
|
+
Requires-Dist: httpx>=0.28.0 ; extra == 'http'
|
|
32
|
+
Requires-Dist: pymongo>=4.11.0 ; extra == 'mongodb'
|
|
33
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.40 ; extra == 'mysql'
|
|
34
|
+
Requires-Dist: asyncmy>=0.2.10 ; extra == 'mysql'
|
|
35
|
+
Requires-Dist: cryptography>=44.0.0 ; extra == 'mysql'
|
|
36
|
+
Requires-Dist: opentelemetry-api>=1.38.0 ; extra == 'observability'
|
|
37
|
+
Requires-Dist: opentelemetry-sdk>=1.38.0 ; extra == 'observability'
|
|
38
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.38.0 ; extra == 'observability'
|
|
39
|
+
Requires-Dist: opentelemetry-instrumentation-fastapi>=0.59b0 ; extra == 'observability'
|
|
40
|
+
Requires-Dist: opentelemetry-instrumentation-httpx>=0.59b0 ; extra == 'observability'
|
|
41
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.40 ; extra == 'postgres'
|
|
42
|
+
Requires-Dist: asyncpg>=0.30.0 ; extra == 'postgres'
|
|
43
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.40 ; extra == 'sqlalchemy'
|
|
44
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.40 ; extra == 'sqlite'
|
|
45
|
+
Requires-Dist: aiosqlite>=0.22.1 ; extra == 'sqlite'
|
|
46
|
+
Requires-Python: >=3.12
|
|
47
|
+
Project-URL: Homepage, https://github.com/VivekReddy10-08-2004/sync-sdk
|
|
48
|
+
Project-URL: Repository, https://github.com/VivekReddy10-08-2004/sync-sdk
|
|
49
|
+
Project-URL: Issues, https://github.com/VivekReddy10-08-2004/sync-sdk/issues
|
|
50
|
+
Project-URL: Test report, https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/TESTING.md
|
|
51
|
+
Project-URL: Changelog, https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/CHANGELOG.md
|
|
52
|
+
Project-URL: Security, https://github.com/VivekReddy10-08-2004/sync-sdk/security/policy
|
|
53
|
+
Provides-Extra: fastapi
|
|
54
|
+
Provides-Extra: gateway
|
|
55
|
+
Provides-Extra: http
|
|
56
|
+
Provides-Extra: mongodb
|
|
57
|
+
Provides-Extra: mysql
|
|
58
|
+
Provides-Extra: observability
|
|
59
|
+
Provides-Extra: postgres
|
|
60
|
+
Provides-Extra: sqlalchemy
|
|
61
|
+
Provides-Extra: sqlite
|
|
62
|
+
Description-Content-Type: text/markdown
|
|
63
|
+
|
|
64
|
+
# Sync SDK
|
|
65
|
+
|
|
66
|
+
Send changes made offline to a central database, retry safely after connection
|
|
67
|
+
failures, and pull server changes back to the device.
|
|
68
|
+
|
|
69
|
+
> **Beta:** Core durability and database guarantees are tested. Public APIs may
|
|
70
|
+
> change before version 1.0.
|
|
71
|
+
|
|
72
|
+
Your application chooses the records, fields, database, authentication, and access
|
|
73
|
+
rules. Sync SDK supplies the durable device outbox, synchronization loop, gateway,
|
|
74
|
+
conflict checks, and database adapters.
|
|
75
|
+
|
|
76
|
+
## Try it locally
|
|
77
|
+
|
|
78
|
+
Run the complete pipeline against temporary SQLite databases:
|
|
79
|
+
|
|
80
|
+
```sh
|
|
81
|
+
uv sync --extra sqlite
|
|
82
|
+
uv run --extra sqlite python examples/pluggable_tasks.py
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The example registers a task table, queues one offline change, synchronizes it
|
|
86
|
+
through an in-process gateway, and prints the device and server records. It removes
|
|
87
|
+
its temporary files when finished. See the
|
|
88
|
+
[example source](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/examples/pluggable_tasks.py).
|
|
89
|
+
|
|
90
|
+
## Install
|
|
91
|
+
|
|
92
|
+
Install the transport, server integration, and database adapter your application
|
|
93
|
+
uses:
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
# Recommended production gateway
|
|
97
|
+
pip install 'offline-sync-sdk[http,fastapi,postgres]'
|
|
98
|
+
|
|
99
|
+
# Other supported gateway databases
|
|
100
|
+
pip install 'offline-sync-sdk[http,fastapi,mysql]'
|
|
101
|
+
pip install 'offline-sync-sdk[http,fastapi,mongodb]'
|
|
102
|
+
|
|
103
|
+
# Local development or a small single-instance gateway
|
|
104
|
+
pip install 'offline-sync-sdk[http,fastapi,sqlite]'
|
|
105
|
+
|
|
106
|
+
# Optional OpenTelemetry support
|
|
107
|
+
pip install 'offline-sync-sdk[http,fastapi,postgres,observability]'
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Python 3.12 or newer is required. The device outbox uses Python's built-in SQLite
|
|
111
|
+
support. You provide the gateway database connection and credentials.
|
|
112
|
+
|
|
113
|
+
## Integration overview
|
|
114
|
+
|
|
115
|
+
| Part | What you add |
|
|
116
|
+
| --- | --- |
|
|
117
|
+
| Server | Register records, build a gateway, and mount its routes. |
|
|
118
|
+
| Device | Open the store, create the client, and enqueue changes. |
|
|
119
|
+
| Worker | Run continuously or schedule one synchronization cycle. |
|
|
120
|
+
|
|
121
|
+
## How it works
|
|
122
|
+
|
|
123
|
+
```mermaid
|
|
124
|
+
flowchart TB
|
|
125
|
+
subgraph device["Device"]
|
|
126
|
+
direction LR
|
|
127
|
+
app["Application"]
|
|
128
|
+
client["SyncClient<br/>enqueue · sync_once · run"]
|
|
129
|
+
local[("SQLiteStore<br/>outbox · synced records<br/>cursor · retry state")]
|
|
130
|
+
local_note["enqueue: outbox only<br/>pull: records + cursor atomically"]
|
|
131
|
+
|
|
132
|
+
app -->|"enqueue mutation"| client
|
|
133
|
+
client -->|"durable enqueue<br/>acknowledge · retry state"| local
|
|
134
|
+
local -->|"FIFO pending batch<br/>cursor"| client
|
|
135
|
+
local -->|"read server-derived state"| app
|
|
136
|
+
local --- local_note
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
subgraph transport["Transport"]
|
|
140
|
+
direction LR
|
|
141
|
+
http["HTTPTransport<br/>push · pull<br/>retry transient failures"]
|
|
142
|
+
network["Network<br/>offline · timeout · lost response"]
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
subgraph host["Host application"]
|
|
146
|
+
direction LR
|
|
147
|
+
access["Authentication<br/>authorization · tenant scope"]
|
|
148
|
+
routes["FastAPI sync routes"]
|
|
149
|
+
gateway["SyncGateway<br/>validation · conflicts · idempotency"]
|
|
150
|
+
|
|
151
|
+
access --> routes --> gateway
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
subgraph server["Server database adapter: PostgreSQL · MySQL · SQLite · MongoDB"]
|
|
155
|
+
direction LR
|
|
156
|
+
transaction["One adapter transaction<br/>whole batch commits or rolls back"]
|
|
157
|
+
records[("Business records<br/>+ sync versions")]
|
|
158
|
+
results[("Processed mutation results<br/>for idempotent replay")]
|
|
159
|
+
feed[("Ordered change feed<br/>+ sequence clock")]
|
|
160
|
+
capture["Optional SQL change capture<br/>direct writes · triggers · cascades"]
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
client <-->|"push batches<br/>pull pages"| http
|
|
164
|
+
http <--> network <--> access
|
|
165
|
+
client -.->|"DirectTransport<br/>(same process)"| gateway
|
|
166
|
+
|
|
167
|
+
gateway --> transaction
|
|
168
|
+
transaction -->|"accepted writes"| records
|
|
169
|
+
transaction -->|"applied · conflict · rejected results"| results
|
|
170
|
+
transaction -->|"accepted changes"| feed
|
|
171
|
+
feed -->|"changes after cursor"| gateway
|
|
172
|
+
records -.->|"poll selected fields"| capture
|
|
173
|
+
capture -.->|"append detected changes"| feed
|
|
174
|
+
|
|
175
|
+
retry["Retryable failures keep mutations pending<br/>stable IDs make replay safe"]
|
|
176
|
+
http --- retry
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
1. The application queues a mutation in the device outbox.
|
|
180
|
+
2. The client sends pending mutations in bounded FIFO batches.
|
|
181
|
+
3. The gateway validates a batch and applies it in one database transaction.
|
|
182
|
+
4. Each mutation result is saved for idempotent replay. Accepted changes also enter
|
|
183
|
+
the ordered server feed.
|
|
184
|
+
5. The device acknowledges outbox results and pulls feed pages after its cursor.
|
|
185
|
+
6. Each page updates local records and its cursor in one SQLite transaction.
|
|
186
|
+
7. After a failure or restart, synchronization resumes from the saved outbox,
|
|
187
|
+
retry state, and cursor.
|
|
188
|
+
|
|
189
|
+
## Add Sync SDK to an application
|
|
190
|
+
|
|
191
|
+
The following path uses PostgreSQL or another SQLAlchemy-supported backend. See
|
|
192
|
+
the [MongoDB guide](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/docs/mongodb.md)
|
|
193
|
+
for document collection mapping.
|
|
194
|
+
|
|
195
|
+
### 1. Register records and fields
|
|
196
|
+
|
|
197
|
+
Use a new SQLAlchemy table or one your application already owns:
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
from sqlalchemy import Column, Integer, MetaData, String, Table
|
|
201
|
+
from sync_sdk.adapters.sqlalchemy_config import SQLAlchemyConfig
|
|
202
|
+
|
|
203
|
+
metadata = MetaData()
|
|
204
|
+
tasks = Table(
|
|
205
|
+
"my_tasks",
|
|
206
|
+
metadata,
|
|
207
|
+
Column("id", Integer, primary_key=True),
|
|
208
|
+
Column("title", String(200), nullable=False),
|
|
209
|
+
Column("private_note", String(200), server_default="private"),
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
config = SQLAlchemyConfig()
|
|
213
|
+
task_binding = config.register_table(
|
|
214
|
+
"tasks",
|
|
215
|
+
tasks,
|
|
216
|
+
fields={"title": "title"},
|
|
217
|
+
)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
`tasks` is the name sent over sync, while `my_tasks` is the database table.
|
|
221
|
+
Only `title` is synchronized; `private_note` stays on the server. If `fields` is
|
|
222
|
+
omitted, every non-key column is included.
|
|
223
|
+
|
|
224
|
+
### 2. Initialize storage and build the gateway
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
|
228
|
+
|
|
229
|
+
engine = create_async_engine("postgresql+asyncpg://user:pass@localhost/app")
|
|
230
|
+
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
|
231
|
+
|
|
232
|
+
await config.create_tables(engine)
|
|
233
|
+
gateway = config.build(sessions)
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`create_tables()` creates registered tables that are missing and the SDK tracking
|
|
237
|
+
tables. It does not alter existing tables. Use application migrations for schema
|
|
238
|
+
changes. If the application already creates its business tables, initialize only
|
|
239
|
+
the SDK metadata:
|
|
240
|
+
|
|
241
|
+
```python
|
|
242
|
+
from sync_sdk.adapters.sqlalchemy import initialize_sync_metadata
|
|
243
|
+
|
|
244
|
+
await initialize_sync_metadata(engine)
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
Run initialization once before starting sync workers.
|
|
248
|
+
|
|
249
|
+
### 3. Mount the gateway routes
|
|
250
|
+
|
|
251
|
+
```python
|
|
252
|
+
from fastapi import FastAPI
|
|
253
|
+
from sync_sdk.integrations.fastapi import create_sync_router
|
|
254
|
+
|
|
255
|
+
app = FastAPI()
|
|
256
|
+
app.include_router(create_sync_router(lambda: gateway, prefix="/v1/sync"))
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
The host application must authenticate these routes and enforce authorization and
|
|
260
|
+
tenant isolation. A gateway exposes the full change feed of its configured store.
|
|
261
|
+
|
|
262
|
+
### 4. Queue and synchronize device changes
|
|
263
|
+
|
|
264
|
+
```python
|
|
265
|
+
import httpx
|
|
266
|
+
from sync_sdk import SQLiteStore, SyncClient
|
|
267
|
+
from sync_sdk.adapters.http import HTTPTransport
|
|
268
|
+
|
|
269
|
+
with SQLiteStore("device.db", client_id="device-1") as local:
|
|
270
|
+
async with httpx.AsyncClient(
|
|
271
|
+
base_url="https://api.example.com",
|
|
272
|
+
timeout=10,
|
|
273
|
+
) as http:
|
|
274
|
+
sync = SyncClient(local, transport=HTTPTransport(http))
|
|
275
|
+
sync.enqueue(
|
|
276
|
+
entity="tasks",
|
|
277
|
+
entity_id="1",
|
|
278
|
+
operation="create",
|
|
279
|
+
payload={"title": "Offline edit"},
|
|
280
|
+
base_version=0,
|
|
281
|
+
)
|
|
282
|
+
report = await sync.sync_once()
|
|
283
|
+
print(report, local.results())
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
Call `enqueue()` once for every application change. Do not enqueue the same logical
|
|
287
|
+
change again after a send failure; the pending mutation keeps its stable ID and is
|
|
288
|
+
retried safely.
|
|
289
|
+
|
|
290
|
+
`sync_once()` performs a bounded amount of push and pull work. Use the continuous
|
|
291
|
+
worker during the application lifetime:
|
|
292
|
+
|
|
293
|
+
```python
|
|
294
|
+
await sync.run(poll_interval=1)
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
Reopen the same SQLite file with the same client ID after a restart. Use one sync
|
|
298
|
+
worker per local file, on the thread that opened it. `DirectTransport(gateway)` can
|
|
299
|
+
replace HTTP when the client and gateway run in the same process.
|
|
300
|
+
|
|
301
|
+
## Consistency and failure behavior
|
|
302
|
+
|
|
303
|
+
| Behavior | Guarantee |
|
|
304
|
+
| --- | --- |
|
|
305
|
+
| Local enqueue | The mutation commits to SQLite before `enqueue()` returns. |
|
|
306
|
+
| Lost response | The mutation remains pending and replays with the same ID. |
|
|
307
|
+
| Duplicate ID | Stored results prevent duplicate application. |
|
|
308
|
+
| Gateway batch | Data and sync metadata commit or roll back together. |
|
|
309
|
+
| Feed order | Sequences are strictly increasing and commit-ordered. |
|
|
310
|
+
| Device checkpoint | Pulled records, deletes, and the cursor commit together. |
|
|
311
|
+
|
|
312
|
+
`base_version` is the last server version known by the application. A different
|
|
313
|
+
server version returns `conflict`; omitting `base_version` skips the default
|
|
314
|
+
version check. Results are `applied`, `conflict`, or `rejected`. The latter two are
|
|
315
|
+
acknowledged and require the application to create a new mutation after resolving
|
|
316
|
+
the problem.
|
|
317
|
+
|
|
318
|
+
Connection failures, HTTP 408/429, and 5xx responses schedule a retry with
|
|
319
|
+
exponential backoff and jitter. `Retry-After` can extend the wait. Other errors are
|
|
320
|
+
raised to the application.
|
|
321
|
+
|
|
322
|
+
The local record view contains changes pulled from the server. `enqueue()` does
|
|
323
|
+
not apply an optimistic update to that view; applications that need immediate
|
|
324
|
+
local UI changes should maintain their own local state.
|
|
325
|
+
|
|
326
|
+
## Capture writes made directly to SQL
|
|
327
|
+
|
|
328
|
+
The optional SQL change-capture worker detects committed inserts, updates, and
|
|
329
|
+
deletes made outside the gateway, including selected trigger effects and cascading
|
|
330
|
+
deletes:
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
import asyncio
|
|
334
|
+
|
|
335
|
+
capture = config.build_change_capture(sessions)
|
|
336
|
+
await capture.capture_once() # establish a baseline; existing rows are not emitted
|
|
337
|
+
capture_task = asyncio.create_task(capture.run(interval=1.0))
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
The worker polls the selected fields of every registered table. Multiple writes to
|
|
341
|
+
one row between scans appear as one final-state change. Large or high-write tables
|
|
342
|
+
may need a native transaction-log CDC adapter instead. MongoDB direct writes are
|
|
343
|
+
not captured by this SQL worker.
|
|
344
|
+
|
|
345
|
+
## Choose a gateway database
|
|
346
|
+
|
|
347
|
+
| Backend | Recommended use | Requirement |
|
|
348
|
+
| --- | --- | --- |
|
|
349
|
+
| PostgreSQL | Production default | Row locking and transactions |
|
|
350
|
+
| MySQL | Existing MySQL deployments | Transactional engine |
|
|
351
|
+
| MongoDB | Document applications | Replica set or sharded cluster |
|
|
352
|
+
| SQLite | Development and small gateways | One local file |
|
|
353
|
+
|
|
354
|
+
Every adapter implements the same gateway contract. SQL and MongoDB adapters
|
|
355
|
+
serialize sequence allocation so a cursor cannot skip a late-committing change.
|
|
356
|
+
This favors correctness over maximum concurrent write throughput.
|
|
357
|
+
|
|
358
|
+
## Performance evidence
|
|
359
|
+
|
|
360
|
+

|
|
361
|
+
|
|
362
|
+
PostgreSQL 16, MySQL 8.4, and MongoDB 8.0 applied all 12,000 measured mutations
|
|
363
|
+
across concurrency levels 1, 2, 4, and 8 with zero failures. Rising latency shows
|
|
364
|
+
batches queuing behind each adapter's shared sequence clock.
|
|
365
|
+
|
|
366
|
+
Run a SQLite smoke benchmark:
|
|
367
|
+
|
|
368
|
+
```sh
|
|
369
|
+
uv run --extra sqlite python benchmarks/sync_throughput.py \
|
|
370
|
+
--mutations 1000 --warmup 100 --batch-size 100 --concurrency 1 \
|
|
371
|
+
--fail-on-errors --output benchmark-sqlite.json
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
The [benchmark guide](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/docs/benchmarks.md)
|
|
375
|
+
defines the metrics, environment, correctness audit, and commands for every
|
|
376
|
+
backend.
|
|
377
|
+
|
|
378
|
+
## Observability
|
|
379
|
+
|
|
380
|
+
Install the `observability` extra and configure OpenTelemetry once at startup:
|
|
381
|
+
|
|
382
|
+
```python
|
|
383
|
+
from sync_sdk.adapters.opentelemetry import setup_signoz
|
|
384
|
+
|
|
385
|
+
monitoring = setup_signoz("my-sync-service")
|
|
386
|
+
sync = SyncClient(local, transport=transport, telemetry=monitoring.telemetry)
|
|
387
|
+
gateway = config.build(sessions, telemetry=monitoring.telemetry)
|
|
388
|
+
```
|
|
389
|
+
|
|
390
|
+
Set `OTEL_EXPORTER_OTLP_ENDPOINT` for the collector and call
|
|
391
|
+
`monitoring.shutdown()` during application shutdown. Monitoring failures do not
|
|
392
|
+
stop synchronization, and payloads or record identifiers are not used as metric
|
|
393
|
+
labels. See the
|
|
394
|
+
[observability guide](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/docs/observability.md)
|
|
395
|
+
for metrics, traces, SigNoz setup, and alert suggestions.
|
|
396
|
+
|
|
397
|
+
## Test and develop
|
|
398
|
+
|
|
399
|
+
Run the normal suite:
|
|
400
|
+
|
|
401
|
+
```sh
|
|
402
|
+
uv sync --locked --group dev --all-extras
|
|
403
|
+
uv run pytest -q
|
|
404
|
+
```
|
|
405
|
+
|
|
406
|
+
GitHub Actions runs the suite on Python 3.12, 3.13, and 3.14, builds and installs
|
|
407
|
+
the package, and verifies live PostgreSQL, MySQL, and MongoDB adapters. The
|
|
408
|
+
[testing report](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/TESTING.md)
|
|
409
|
+
records exact versions, commands, expected skips, measured results, and release
|
|
410
|
+
checks.
|
|
411
|
+
|
|
412
|
+
The `app/` directory contains a sample FastAPI gateway:
|
|
413
|
+
|
|
414
|
+
```sh
|
|
415
|
+
uv run --extra gateway alembic upgrade head
|
|
416
|
+
uv run --extra gateway uvicorn app.main:app --reload
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
## Current scope
|
|
420
|
+
|
|
421
|
+
- The host application owns authentication, authorization, tenant isolation,
|
|
422
|
+
database credentials, schema migrations, backups, and retention.
|
|
423
|
+
- Existing server rows are not emitted automatically. The first SQL capture scan
|
|
424
|
+
records them as its baseline.
|
|
425
|
+
- SQL change capture scans registered tables; native transaction-log CDC is not
|
|
426
|
+
included.
|
|
427
|
+
- MongoDB is the ready-to-use NoSQL adapter. Its gateway writes require
|
|
428
|
+
multi-document transaction support.
|
|
429
|
+
- The SDK provides at-least-once delivery attempts with idempotent server
|
|
430
|
+
processing; permanent validation or authorization errors require application
|
|
431
|
+
handling.
|
|
432
|
+
|
|
433
|
+
For composite keys, linked tables, custom schemas, custom adapters, and metadata
|
|
434
|
+
tables, see the
|
|
435
|
+
[setup and adapter guide](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/docs/approach_architecture.md).
|
|
436
|
+
|
|
437
|
+
## Contributing and security
|
|
438
|
+
|
|
439
|
+
See [CONTRIBUTING.md](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/CONTRIBUTING.md)
|
|
440
|
+
for development setup and DCO sign-off. Report vulnerabilities privately through
|
|
441
|
+
the process in
|
|
442
|
+
[SECURITY.md](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/SECURITY.md).
|
|
443
|
+
|
|
444
|
+
## License
|
|
445
|
+
|
|
446
|
+
Sync SDK is available under the
|
|
447
|
+
[MIT License](https://github.com/VivekReddy10-08-2004/sync-sdk/blob/master/LICENSE).
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
sync_sdk/__init__.py,sha256=neeZVHqGobgxIZ4OT8-qLsuh0Jequm3cHMhvWq3Fcx0,885
|
|
2
|
+
sync_sdk/adapters/__init__.py,sha256=0AhHIadic6pincOlKYeBVNZPWNLYi8pfVjV3ik-Nk8o,75
|
|
3
|
+
sync_sdk/adapters/direct.py,sha256=qqGYLyFUpTHb3aNWfcsGKMI1sGV4Lrs-Iej8hr_O-Nw,518
|
|
4
|
+
sync_sdk/adapters/http.py,sha256=zyXtTOpj00MIulhw2zQy1l1krycCrHkRO-qi3dSkIcI,2586
|
|
5
|
+
sync_sdk/adapters/mongodb.py,sha256=BDpzyorSsdARLKWMA0mIvkrmDR0ZHYL2u9w94Kzej14,17679
|
|
6
|
+
sync_sdk/adapters/opentelemetry.py,sha256=kDPAJcK78xsBeNs6q46uybswsxtdJz-OqiJYLY5VDyg,8727
|
|
7
|
+
sync_sdk/adapters/sqlalchemy.py,sha256=ZeZlyF9KaI1H1HAy1bAfOgCBEbA08QJwmF9vpKbM6wQ,26810
|
|
8
|
+
sync_sdk/adapters/sqlalchemy_config.py,sha256=us50o2RLN-HygSBxJvcospQs1rjvEfPoCNi7Pc2Mrzk,5818
|
|
9
|
+
sync_sdk/client.py,sha256=s3cJz1g7mzwXTqDCopZpa-vY0mb9I7bQKiR4uexa8tg,8318
|
|
10
|
+
sync_sdk/gateway.py,sha256=hs9sbWNEC9sq6xUIU2g0CYTEHKKXomgK2wlCaXnTro4,6005
|
|
11
|
+
sync_sdk/integrations/__init__.py,sha256=rzBqb7zhYSIPwdn7Ax9BB9hjnH7MhmTXPNZpIQw8-4k,51
|
|
12
|
+
sync_sdk/integrations/fastapi.py,sha256=K-U9qzphvEf_AgJMGtiW43HhEElh9hWJ5c-012q5N8Y,1078
|
|
13
|
+
sync_sdk/models.py,sha256=Nx8fRlr41EH3inFAW9pxvGkZPsy0htCoa5kdgd4lIX0,930
|
|
14
|
+
sync_sdk/ports.py,sha256=38Ayc7Cd-vj86Zkjkv28fvwAUPaOA84IfHxVUxKNbyM,3485
|
|
15
|
+
sync_sdk/py.typed,sha256=bWew9mHgMy8LqMu7RuqQXFXLBxh2CRx0dUbSx-3wE48,27
|
|
16
|
+
sync_sdk/registry.py,sha256=XYyAtWZJDJdG7IO9smYrWFBbCtcQ1HFydqLn1kzg970,1465
|
|
17
|
+
sync_sdk/store.py,sha256=MUpfJ4N856DzQqr8wMDgtKGR71_lc89B4BLcn-XUAA4,5800
|
|
18
|
+
sync_sdk/telemetry.py,sha256=s_lf8CWynGcIaRBxSH9BlFLoBcDKB1LrfkP5w4_bFu0,3970
|
|
19
|
+
offline_sync_sdk-0.2.0.dist-info/WHEEL,sha256=-i9oRNYVXXZJUIYl5zclLIg6onEb0NLibTX34uln84w,81
|
|
20
|
+
offline_sync_sdk-0.2.0.dist-info/entry_points.txt,sha256=xyXXQrbZWySQqUbWcqCuMJNb4JDNx9EUGXujNASJ0U4,44
|
|
21
|
+
offline_sync_sdk-0.2.0.dist-info/METADATA,sha256=8al8mH7Qqo69wcHmHfwg4BX9erIYvwBP9VGDk5IbJmQ,17658
|
|
22
|
+
offline_sync_sdk-0.2.0.dist-info/RECORD,,
|
sync_sdk/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from .client import SyncClient, SyncReport
|
|
2
|
+
from .gateway import SyncGateway
|
|
3
|
+
from .models import Change, ChangeBatch, Mutation, MutationResult
|
|
4
|
+
from .ports import (
|
|
5
|
+
GatewayStore,
|
|
6
|
+
GatewayTransaction,
|
|
7
|
+
LocalStore,
|
|
8
|
+
MutationRejected,
|
|
9
|
+
RetryableError,
|
|
10
|
+
SyncTransport,
|
|
11
|
+
)
|
|
12
|
+
from .registry import Entity, EntityRegistry
|
|
13
|
+
from .store import SQLiteStore
|
|
14
|
+
from .telemetry import NoOpTelemetry, PipelineTelemetry
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Change",
|
|
18
|
+
"ChangeBatch",
|
|
19
|
+
"Entity",
|
|
20
|
+
"EntityRegistry",
|
|
21
|
+
"GatewayStore",
|
|
22
|
+
"GatewayTransaction",
|
|
23
|
+
"LocalStore",
|
|
24
|
+
"Mutation",
|
|
25
|
+
"MutationRejected",
|
|
26
|
+
"MutationResult",
|
|
27
|
+
"NoOpTelemetry",
|
|
28
|
+
"PipelineTelemetry",
|
|
29
|
+
"RetryableError",
|
|
30
|
+
"SQLiteStore",
|
|
31
|
+
"SyncClient",
|
|
32
|
+
"SyncGateway",
|
|
33
|
+
"SyncReport",
|
|
34
|
+
"SyncTransport",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main() -> None:
|
|
39
|
+
print("Use SyncClient and SQLiteStore to connect to a sync gateway.")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Opt-in adapters. Import the specific module for the backend you use."""
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Embed the service without HTTP or a web framework."""
|
|
2
|
+
|
|
3
|
+
from ..models import ChangeBatch, Mutation, MutationResult
|
|
4
|
+
from ..ports import RemoteStore
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DirectTransport:
|
|
8
|
+
def __init__(self, gateway: RemoteStore):
|
|
9
|
+
self.gateway = gateway
|
|
10
|
+
|
|
11
|
+
async def push(self, mutations: list[Mutation]) -> list[MutationResult]:
|
|
12
|
+
return await self.gateway.apply_mutations(mutations)
|
|
13
|
+
|
|
14
|
+
async def pull(self, cursor: int, limit: int) -> ChangeBatch:
|
|
15
|
+
return await self.gateway.get_changes(cursor, limit)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""HTTP transport; HTTP-specific failures stay outside the sync engine."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from email.utils import parsedate_to_datetime
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
from pydantic import TypeAdapter
|
|
10
|
+
|
|
11
|
+
from ..models import ChangeBatch, Mutation, MutationResult
|
|
12
|
+
from ..ports import RetryableError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class HTTPTransport:
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
client: httpx.AsyncClient,
|
|
19
|
+
*,
|
|
20
|
+
prefix: str = "/v1/sync",
|
|
21
|
+
clock: Callable[[], float] = time.time,
|
|
22
|
+
):
|
|
23
|
+
self.client, self.prefix, self.clock = client, prefix.rstrip("/"), clock
|
|
24
|
+
|
|
25
|
+
def _retry_after(self, response: httpx.Response) -> float:
|
|
26
|
+
value = response.headers.get("Retry-After", "")
|
|
27
|
+
try:
|
|
28
|
+
delay = float(value)
|
|
29
|
+
except ValueError:
|
|
30
|
+
try:
|
|
31
|
+
delay = parsedate_to_datetime(value).timestamp() - self.clock()
|
|
32
|
+
except (ValueError, TypeError, OverflowError):
|
|
33
|
+
return 0
|
|
34
|
+
return max(0, delay) if math.isfinite(delay) else 0
|
|
35
|
+
|
|
36
|
+
async def _request(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
37
|
+
try:
|
|
38
|
+
response = await self.client.request(method, self.prefix + path, **kwargs)
|
|
39
|
+
response.raise_for_status()
|
|
40
|
+
return response
|
|
41
|
+
except httpx.TransportError as exc:
|
|
42
|
+
raise RetryableError(str(exc), reason="connection") from exc
|
|
43
|
+
except httpx.HTTPStatusError as exc:
|
|
44
|
+
if (
|
|
45
|
+
exc.response.status_code in (408, 429)
|
|
46
|
+
or exc.response.status_code >= 500
|
|
47
|
+
):
|
|
48
|
+
if exc.response.status_code == 408:
|
|
49
|
+
reason = "timeout"
|
|
50
|
+
elif exc.response.status_code == 429:
|
|
51
|
+
reason = "rate_limit"
|
|
52
|
+
else:
|
|
53
|
+
reason = "server"
|
|
54
|
+
raise RetryableError(
|
|
55
|
+
str(exc),
|
|
56
|
+
retry_after=self._retry_after(exc.response),
|
|
57
|
+
reason=reason,
|
|
58
|
+
) from exc
|
|
59
|
+
raise
|
|
60
|
+
|
|
61
|
+
async def push(self, mutations: list[Mutation]) -> list[MutationResult]:
|
|
62
|
+
response = await self._request(
|
|
63
|
+
"POST", "/mutations", json=[m.model_dump(mode="json") for m in mutations]
|
|
64
|
+
)
|
|
65
|
+
return TypeAdapter(list[MutationResult]).validate_python(response.json())
|
|
66
|
+
|
|
67
|
+
async def pull(self, cursor: int, limit: int) -> ChangeBatch:
|
|
68
|
+
response = await self._request(
|
|
69
|
+
"GET", "/changes", params={"cursor": cursor, "limit": limit}
|
|
70
|
+
)
|
|
71
|
+
return ChangeBatch.model_validate(response.json())
|