fastapi-offline-sync 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastapi_offline_sync-0.1.1/PKG-INFO +206 -0
- fastapi_offline_sync-0.1.1/PRD.MD +408 -0
- fastapi_offline_sync-0.1.1/README.md +169 -0
- fastapi_offline_sync-0.1.1/client-prd-updated.md +359 -0
- fastapi_offline_sync-0.1.1/examples/app.py +57 -0
- fastapi_offline_sync-0.1.1/pyproject.toml +74 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/__init__.py +5 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/cli.py +26 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/config.py +90 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/exceptions.py +29 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/hlc.py +92 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/metrics.py +39 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/mongo.py +55 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/resolver.py +64 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/router.py +138 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/schemas/__init__.py +20 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/schemas/common.py +42 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/schemas/full.py +22 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/schemas/pull.py +34 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/schemas/push.py +36 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/schemas/stream.py +30 -0
- fastapi_offline_sync-0.1.1/src/fastapi_offline_sync/service.py +562 -0
- fastapi_offline_sync-0.1.1/tests/test_hlc.py +74 -0
- fastapi_offline_sync-0.1.1/tests/test_router.py +87 -0
- fastapi_offline_sync-0.1.1/tests/test_schemas.py +31 -0
- fastapi_offline_sync-0.1.1/tests/test_service.py +770 -0
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fastapi-offline-sync
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Offline-first sync primitives for FastAPI and MongoDB applications.
|
|
5
|
+
Author: Fisco Team
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: fastapi,indexeddb,mongodb,offline-first,react-native,sync
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Framework :: FastAPI
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Requires-Dist: fastapi>=0.136.1
|
|
22
|
+
Requires-Dist: motor>=3.7.1
|
|
23
|
+
Requires-Dist: prometheus-client>=0.25.0
|
|
24
|
+
Requires-Dist: prometheus-fastapi-instrumentator>=7.1.0
|
|
25
|
+
Requires-Dist: pydantic-settings>=2.14.0
|
|
26
|
+
Requires-Dist: pydantic>=2.13.3
|
|
27
|
+
Requires-Dist: pymongo>=4.17.0
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: build>=1.3.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: httpx>=0.28.1; extra == 'dev'
|
|
31
|
+
Requires-Dist: mypy>=1.20.2; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest-asyncio>=1.3.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest>=9.0.3; extra == 'dev'
|
|
34
|
+
Requires-Dist: ruff>=0.15.12; extra == 'dev'
|
|
35
|
+
Requires-Dist: twine>=6.1.0; extra == 'dev'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# fastapi-offline-sync
|
|
39
|
+
|
|
40
|
+
`fastapi-offline-sync` is a robust, production-ready offline-first sync layer for FastAPI applications backed by MongoDB. It provides high-performance synchronization endpoints and live WebSocket updates with built-in conflict resolution, multi-worker safety, and atomic transactions.
|
|
41
|
+
|
|
42
|
+
## Features
|
|
43
|
+
|
|
44
|
+
- **Incremental Pull**: Efficient retrieval of incremental data changes using Hybrid Logical Clocks (HLC).
|
|
45
|
+
- **Atomic Push Batches**: Safely apply client mutations in atomic MongoDB transactions, preventing partial state corruption.
|
|
46
|
+
- **WebSocket Streaming**: Live Change Stream propagation for real-time document updates.
|
|
47
|
+
- **Distributed Multi-Worker Safety**: Custom HLC node namespaces derived automatically or configured via container metadata to eliminate collision risks.
|
|
48
|
+
- **Soft-Delete Propagation**: Seamless tombstones on full resyncs to clear obsolete client-side records.
|
|
49
|
+
|
|
50
|
+
## Installation
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install fastapi-offline-sync
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Quick Start
|
|
57
|
+
|
|
58
|
+
Initialize and mount the router with production configurations:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import os
|
|
62
|
+
from fastapi import FastAPI
|
|
63
|
+
from fastapi_offline_sync import SyncConfig, SyncRouter
|
|
64
|
+
|
|
65
|
+
app = FastAPI(title="Production Sync Service")
|
|
66
|
+
|
|
67
|
+
config = SyncConfig(
|
|
68
|
+
mongodb_uri=os.environ["MONGODB_URI"],
|
|
69
|
+
database_name=os.environ["MONGODB_DATABASE"],
|
|
70
|
+
collections=("tasks", "inventory_items"),
|
|
71
|
+
# Set unique HLC node ID (e.g., container hostname / Pod IP) for multi-worker safety
|
|
72
|
+
hlc_node_id=os.environ.get("HOSTNAME"),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
app.include_router(SyncRouter(config))
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Authentication & Identity Resolution
|
|
79
|
+
|
|
80
|
+
Configure a JWT dependency to verify credentials and return authenticated user properties.
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from fastapi import Header, HTTPException, status
|
|
84
|
+
|
|
85
|
+
async def verify_jwt(
|
|
86
|
+
authorization: str | None = Header(default=None),
|
|
87
|
+
) -> dict[str, str]:
|
|
88
|
+
if not authorization or not authorization.startswith("Bearer "):
|
|
89
|
+
raise HTTPException(
|
|
90
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
91
|
+
detail="Missing or invalid authentication header",
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
token = authorization.removeprefix("Bearer ")
|
|
95
|
+
# Decode and verify the JWT with your auth provider
|
|
96
|
+
# Extract identity fields:
|
|
97
|
+
return {
|
|
98
|
+
"user_id": "user-unique-identifier",
|
|
99
|
+
"business_id": "business-tenant-identifier"
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Then register the dependency with `SyncConfig`:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
config = SyncConfig(
|
|
107
|
+
mongodb_uri=os.environ["MONGODB_URI"],
|
|
108
|
+
database_name=os.environ["MONGODB_DATABASE"],
|
|
109
|
+
collections=("tasks",),
|
|
110
|
+
jwt_dependency=verify_jwt,
|
|
111
|
+
)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Fisco Integration
|
|
115
|
+
|
|
116
|
+
For Fisco-style backends where data is scoped by business/tenant rather than individual user, configure the sync engine to scope operations by `business_id` while keeping the acting identity as `user_id`.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from fastapi_offline_sync import SyncConfig, SyncRouter, SyncService
|
|
120
|
+
|
|
121
|
+
config = SyncConfig(
|
|
122
|
+
mongodb_uri=os.environ["MONGODB_URI"],
|
|
123
|
+
database_name="Fisco",
|
|
124
|
+
collections=(
|
|
125
|
+
"inventory_items",
|
|
126
|
+
"categories",
|
|
127
|
+
"orders",
|
|
128
|
+
"customers",
|
|
129
|
+
"sales",
|
|
130
|
+
),
|
|
131
|
+
actor_id_field="user_id",
|
|
132
|
+
scope_id_field="business_id",
|
|
133
|
+
jwt_dependency=verify_jwt,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
router = SyncRouter(config)
|
|
137
|
+
sync_service = SyncService(config)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Server-Layer Writes
|
|
141
|
+
|
|
142
|
+
For existing service-layer database writes, ensure that you append HLC metadata and oplog entries in the same session:
|
|
143
|
+
|
|
144
|
+
1. Retrieve sync metadata with `sync_service.build_sync_metadata(actor=user, scope_id=business_id)`.
|
|
145
|
+
2. Persist the metadata fields `_sync_version`, `_sync_actor_id`, `_sync_scope_id`, and `_sync_deleted` with the document.
|
|
146
|
+
3. Record the change using `sync_service.record_server_change(...)`.
|
|
147
|
+
|
|
148
|
+
## Deploying to Production
|
|
149
|
+
|
|
150
|
+
Run the application using a production-grade ASGI server:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
uvicorn examples.app:app --host 0.0.0.0 --port 8000 --workers 4
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## API Reference
|
|
157
|
+
|
|
158
|
+
### 1. Push Client Changes (`POST /sync/push`)
|
|
159
|
+
Pushes client-side mutations to the server.
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
curl -X POST https://api.yourservice.com/sync/push \
|
|
163
|
+
-H 'Content-Type: application/json' \
|
|
164
|
+
-H 'Authorization: Bearer <JWT_TOKEN>' \
|
|
165
|
+
-d '{
|
|
166
|
+
"client_id": "device-client-uuid",
|
|
167
|
+
"changes": [
|
|
168
|
+
{
|
|
169
|
+
"collection": "tasks",
|
|
170
|
+
"operation": "upsert",
|
|
171
|
+
"doc_id": "task-abc-123",
|
|
172
|
+
"data": {"title": "Acquire inventory", "done": false},
|
|
173
|
+
"parent_version": "20260502T120000.000Z-0001-a1b2"
|
|
174
|
+
}
|
|
175
|
+
]
|
|
176
|
+
}'
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### 2. Incremental Pull (`GET /sync/pull`)
|
|
180
|
+
Fetches operations since a specific HLC token.
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
curl -H 'Authorization: Bearer <JWT_TOKEN>' \
|
|
184
|
+
'https://api.yourservice.com/sync/pull?since=20260502T120000.000Z-0001-a1b2&collections=tasks&limit=100'
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
### 3. Full Resync (`GET /sync/full`)
|
|
188
|
+
Triggers a complete sync with Gzip compression and tombstones for soft-deleted documents.
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
curl -H 'Authorization: Bearer <JWT_TOKEN>' \
|
|
192
|
+
--compressed \
|
|
193
|
+
'https://api.yourservice.com/sync/full?collections=tasks&limit=1000'
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### 4. Live Updates WebSocket (`WS /sync/stream`)
|
|
197
|
+
Subscribes to live database modifications via a persistent WebSocket connection.
|
|
198
|
+
|
|
199
|
+
```json
|
|
200
|
+
{
|
|
201
|
+
"type": "subscribe",
|
|
202
|
+
"since": "20260502T120000.000Z-0001-a1b2",
|
|
203
|
+
"collections": ["tasks"],
|
|
204
|
+
"token": "<JWT_TOKEN>"
|
|
205
|
+
}
|
|
206
|
+
```
|
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
Here’s the **Server‑Side PRD** for the `fastapi-offline-sync` package, extracted and detailed from the full offline sync engine. This focuses exclusively on the Python/FastAPI backend component.
|
|
2
|
+
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# Product Requirements Document
|
|
6
|
+
## `fastapi-offline-sync` (Server‑Side Package)
|
|
7
|
+
|
|
8
|
+
| Version | Date | Author | Status |
|
|
9
|
+
|---------|------------|---------------|------------|
|
|
10
|
+
| 1.0 | 2026-05-02 | Fisco Team | Draft |
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## 1. Introduction
|
|
15
|
+
|
|
16
|
+
### 1.1 Purpose
|
|
17
|
+
`fastapi-offline-sync` is a Python library that adds offline‑first synchronization capabilities to any FastAPI application backed by MongoDB. It provides ready‑to‑use REST and WebSocket endpoints to accept, resolve, and redistribute data changes from intermittently connected clients. The package handles conflict resolution, change tracking, and fallback strategies, enabling developers to build offline‑resilient apps without reinventing the sync wheel.
|
|
18
|
+
|
|
19
|
+
### 1.2 Problem Statement
|
|
20
|
+
Offline‑first mobile and web applications require a backend that can:
|
|
21
|
+
- Accept batched changes from clients that have been offline for days or weeks.
|
|
22
|
+
- Resolve conflicts deterministically without user intervention (or with customizable logic).
|
|
23
|
+
- Serve a reliable, incremental stream of changes to bring stale clients up to date.
|
|
24
|
+
- Operate efficiently even under intermittent, low‑bandwidth connections typical in regions like Nigeria.
|
|
25
|
+
|
|
26
|
+
Existing sync solutions mostly target Node.js backends or require specific database adapters. This package gives the Python/FastAPI ecosystem a native, production‑ready answer.
|
|
27
|
+
|
|
28
|
+
### 1.3 Target Audience
|
|
29
|
+
- Python developers building FastAPI + MongoDB applications that need offline support for web and React Native clients.
|
|
30
|
+
- Open‑source contributors wanting to extend the FastAPI ecosystem with an offline sync layer.
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## 2. Project Goals
|
|
35
|
+
|
|
36
|
+
1. **Drop‑in integration**: add sync endpoints with minimal configuration (5 lines or less).
|
|
37
|
+
2. **Reliable offline sync**: handle large backlogs, long offline gaps, and conflict storms.
|
|
38
|
+
3. **Deterministic, configurable conflict resolution**: sensible defaults that can be overridden per collection.
|
|
39
|
+
4. **Efficient delta sync**: serve only changes since the client’s last known version.
|
|
40
|
+
5. **Graceful fallback**: when a client is too stale, provide a full data resync without data loss.
|
|
41
|
+
6. **Real‑time live sync**: WebSocket support for instantly pushing changes to connected clients.
|
|
42
|
+
7. **Observable, secure, and scalable**: metrics, authentication hooks, and horizontal scalability guidance.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## 3. Scope
|
|
47
|
+
|
|
48
|
+
### In Scope
|
|
49
|
+
- FastAPI router (`SyncRouter`) with:
|
|
50
|
+
- `POST /sync/push` – accept client changes.
|
|
51
|
+
- `GET /sync/pull` – deliver incremental changes.
|
|
52
|
+
- `GET /sync/full` – deliver complete dataset for stale clients.
|
|
53
|
+
- `WS /sync/stream` – real‑time change feed.
|
|
54
|
+
- Hybrid Logical Clock (HLC) generation and ordering.
|
|
55
|
+
- MongoDB oplog collection (`sync_oplog`) with TTL management.
|
|
56
|
+
- Conflict resolution engine with default Last‑Writer‑Wins (LWW) and pluggable custom resolvers.
|
|
57
|
+
- Per‑collection policies for delete‑update scenarios.
|
|
58
|
+
- Authentication integration via FastAPI dependency injection (JWT scopes).
|
|
59
|
+
- Full resync mechanism when client `since` is beyond oplog retention.
|
|
60
|
+
- Prometheus metrics endpoint for sync operations.
|
|
61
|
+
- Python type hints and auto‑generated OpenAPI documentation.
|
|
62
|
+
|
|
63
|
+
### Out of Scope (Future)
|
|
64
|
+
- Built‑in CRDT data types (Yjs/Automerge) – may be offered as optional extensions.
|
|
65
|
+
- Peer‑to‑peer sync.
|
|
66
|
+
- Client libraries.
|
|
67
|
+
- Admin UI.
|
|
68
|
+
- GraphQL subscription support.
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## 4. Functional Requirements
|
|
73
|
+
|
|
74
|
+
### FR1 – `POST /sync/push` – Client Change Ingestion
|
|
75
|
+
|
|
76
|
+
**FR1.1** The endpoint accepts a JSON body with:
|
|
77
|
+
- `client_id`: string (device identifier).
|
|
78
|
+
- `changes`: array of change objects.
|
|
79
|
+
|
|
80
|
+
Each change object:
|
|
81
|
+
- `collection`: string (MongoDB collection name).
|
|
82
|
+
- `operation`: `"upsert"` | `"delete"`.
|
|
83
|
+
- `doc_id`: string or ObjectId.
|
|
84
|
+
- `data`: object (null for delete).
|
|
85
|
+
- `parent_version`: string (HLC version the change was based on).
|
|
86
|
+
|
|
87
|
+
**FR1.2** The server processes each change sequentially, ensuring the following **atomic steps per change** (using MongoDB transactions):
|
|
88
|
+
1. Fetch the current document from the business collection.
|
|
89
|
+
2. Compare `parent_version` with the current document’s version (stored in a reserved `_sync_version` field or obtained from the oplog).
|
|
90
|
+
3. If versions match → fast path: apply the change.
|
|
91
|
+
4. If versions differ → invoke the conflict resolver for that collection.
|
|
92
|
+
5. Generate a new HLC version.
|
|
93
|
+
6. Store the resolved document in the business collection, setting `_sync_version` and `_sync_user_id`.
|
|
94
|
+
7. Insert a new entry in `sync_oplog` with the new version, the final document snapshot, and metadata (`operation`, `collection`, `doc_id`, `user_id`, `parent_version`).
|
|
95
|
+
8. Return a success or conflict status for the change.
|
|
96
|
+
|
|
97
|
+
**FR1.3** The response contains an array of results, each with:
|
|
98
|
+
- `doc_id`: string.
|
|
99
|
+
- `status`: `"accepted"` | `"conflict_resolved"` | `"rejected"`.
|
|
100
|
+
- `new_version`: (HLC string) if accepted or resolved.
|
|
101
|
+
- `error`: optional message for rejection.
|
|
102
|
+
|
|
103
|
+
**FR1.4** The endpoint requires a valid JWT (configurable via dependency). The authenticated user’s ID is used for `_sync_user_id` and may be compared against a user‑specific visibility scope (e.g., only own documents).
|
|
104
|
+
|
|
105
|
+
**FR1.5** The server must handle partial batches: if one change fails validation, the whole batch may be rolled back (configurable) or individual failures returned, leaving successful changes applied.
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
### FR2 – `GET /sync/pull` – Incremental Change Feed
|
|
110
|
+
|
|
111
|
+
**FR2.1** Query parameters:
|
|
112
|
+
- `since` (required): HLC version to start pulling from (exclusive).
|
|
113
|
+
- `collections` (optional): comma‑separated list of collection names to filter.
|
|
114
|
+
- `limit` (optional, default 500, max 1000): max changes returned.
|
|
115
|
+
- `user_id` (implicit from JWT, used to filter oplog entries).
|
|
116
|
+
|
|
117
|
+
**FR2.2** The server queries `sync_oplog` for entries with `_id > since`, filtered by `user_id` (and optionally `collections`), sorted by `_id` ascending, limited by `limit`.
|
|
118
|
+
|
|
119
|
+
**FR2.3** Response JSON:
|
|
120
|
+
```json
|
|
121
|
+
{
|
|
122
|
+
"changes": [
|
|
123
|
+
{
|
|
124
|
+
"version": "<hlc>",
|
|
125
|
+
"collection": "tasks",
|
|
126
|
+
"doc_id": "abc123",
|
|
127
|
+
"operation": "upsert",
|
|
128
|
+
"data": { ... }
|
|
129
|
+
}
|
|
130
|
+
],
|
|
131
|
+
"last_seq": "<hlc>" // the version of the last change in this batch, or the input `since` if empty
|
|
132
|
+
}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**FR2.4** If `since` is older than the earliest oplog entry (i.e., older than the TTL), the server returns:
|
|
136
|
+
```json
|
|
137
|
+
{
|
|
138
|
+
"full_resync_required": true,
|
|
139
|
+
"collections": ["tasks", "projects"] // affected collections from the request
|
|
140
|
+
}
|
|
141
|
+
```
|
|
142
|
+
The client should then call the full resync endpoint.
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
### FR3 – `GET /sync/full` – Full Resync (Stale Client Fallback)
|
|
147
|
+
|
|
148
|
+
**FR3.1** Query parameters:
|
|
149
|
+
- `collections` (required): comma‑separated list of collections to export.
|
|
150
|
+
- `cursor` (optional): pagination cursor (opaque string) for large datasets.
|
|
151
|
+
- `limit` (optional, default 1000): batch size.
|
|
152
|
+
|
|
153
|
+
**FR3.2** The server streams the current state of the requested collections as **Newline‑Delimited JSON (NDJSON)** with gzip compression (`Accept-Encoding: gzip`). Each line is a JSON object with `collection`, `doc_id`, and `data`.
|
|
154
|
+
|
|
155
|
+
**FR3.3** Response headers include:
|
|
156
|
+
- `X-Sync-Cursor`: cursor to request the next batch (if any remaining).
|
|
157
|
+
- `X-Sync-LastSeq`: the latest HLC version of the included data (client will use as new `since` after full resync).
|
|
158
|
+
- `Content-Type: application/x-ndjson`.
|
|
159
|
+
|
|
160
|
+
**FR3.4** The endpoint must be efficient and not cause memory issues for large collections (use server‑side cursors).
|
|
161
|
+
|
|
162
|
+
**FR3.5** Authentication and user scoping must still apply (only export documents the user is allowed to see).
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
### FR4 – `WS /sync/stream` – Real‑Time Sync
|
|
167
|
+
|
|
168
|
+
**FR4.1** On connection, client sends a subscription message:
|
|
169
|
+
```json
|
|
170
|
+
{
|
|
171
|
+
"type": "subscribe",
|
|
172
|
+
"since": "<hlc>",
|
|
173
|
+
"collections": ["tasks", "projects"]
|
|
174
|
+
}
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
**FR4.2** Server validates JWT from the WebSocket handshake (token passed as query parameter or in initial message).
|
|
178
|
+
|
|
179
|
+
**FR4.3** Server begins monitoring the `sync_oplog` collection using MongoDB Change Streams, filtered to the user’s data and requested collections.
|
|
180
|
+
|
|
181
|
+
**FR4.4** For each new oplog entry, the server pushes a JSON frame to the client:
|
|
182
|
+
```json
|
|
183
|
+
{
|
|
184
|
+
"version": "<hlc>",
|
|
185
|
+
"collection": "tasks",
|
|
186
|
+
"doc_id": "abc123",
|
|
187
|
+
"operation": "upsert",
|
|
188
|
+
"data": { ... }
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**FR4.5** Server must handle disconnections gracefully (no crash). A heartbeat/ping every 30 seconds keeps the connection alive.
|
|
193
|
+
|
|
194
|
+
**FR4.6** If the oplog TTL has purged the requested `since`, the server sends:
|
|
195
|
+
```json
|
|
196
|
+
{
|
|
197
|
+
"type": "full_resync_required",
|
|
198
|
+
"collections": ["tasks", "projects"]
|
|
199
|
+
}
|
|
200
|
+
```
|
|
201
|
+
and then closes the connection (client falls back to HTTP full resync).
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
### FR5 – Conflict Resolution Engine
|
|
206
|
+
|
|
207
|
+
**FR5.1** The library includes a default conflict resolver `LastWriterWinsByHLC`. It compares the HLC of the incoming change’s parent version against the current document’s version; the change with the higher HLC wins. In case of identical HLC (rare), a tie‑breaking rule (e.g., lexicographic `doc_id`) is used.
|
|
208
|
+
|
|
209
|
+
**FR5.2** Developers can provide a custom resolver function per collection via configuration:
|
|
210
|
+
```python
|
|
211
|
+
def custom_resolver(collection: str, doc_id: Any, current_doc: Optional[Dict],
|
|
212
|
+
incoming_change: Dict, current_version: str, incoming_parent_version: str) -> Dict:
|
|
213
|
+
# Return final document to store, or raise ConflictRejection
|
|
214
|
+
...
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
**FR5.3** For delete‑update conflicts, a per‑collection policy can be set:
|
|
218
|
+
- `resurrect_on_update` (default): if a client updates a deleted document, the document is recreated with the new data.
|
|
219
|
+
- `reject_update`: the server rejects the update and returns a permanent conflict (the client must discard its change).
|
|
220
|
+
- `delete_wins`: the update is ignored, and the document remains deleted (soft‑delete marker stays).
|
|
221
|
+
|
|
222
|
+
**FR5.4** The conflict resolver is invoked inside the transaction that applies the change. It must be deterministic and fast (<10 ms typical).
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
### FR6 – Authentication & Authorization
|
|
227
|
+
|
|
228
|
+
**FR6.1** All sync endpoints accept a FastAPI dependency injection for authentication (e.g., `Depends(get_current_user)`). The dependency returns a user object with at least `user_id`.
|
|
229
|
+
|
|
230
|
+
**FR6.2** Optional: a collection‑scoped authorization callback can be registered. The server calls it before allowing push/pull/full for a specific collection, passing `user` and `collection` name. If it returns `False`, the operation is denied (403).
|
|
231
|
+
|
|
232
|
+
**FR6.3** The `sync_oplog` entry includes `user_id` so that the pull endpoint can filter changes to only those visible to the requesting user (multi‑tenant safety).
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
### FR7 – Oplog & Database Management
|
|
237
|
+
|
|
238
|
+
**FR7.1** The package creates a MongoDB collection named `sync_oplog` with the following schema:
|
|
239
|
+
- `_id`: string (HLC version, e.g., `"20260502T143000.000Z-0001"`)
|
|
240
|
+
- `timestamp`: datetime (used for TTL index)
|
|
241
|
+
- `collection`: string
|
|
242
|
+
- `doc_id`: any
|
|
243
|
+
- `operation`: string (`upsert` / `delete`)
|
|
244
|
+
- `data`: object (full document snapshot for upsert, null for delete)
|
|
245
|
+
- `user_id`: string
|
|
246
|
+
- `parent_version`: string (nullable)
|
|
247
|
+
|
|
248
|
+
**FR7.2** A TTL index is created on `timestamp` with an expiry of `SYNC_OPLOG_TTL_DAYS` (configurable, default 30 days). This ensures the oplog doesn’t grow unbounded.
|
|
249
|
+
|
|
250
|
+
**FR7.3** The business collections are augmented with a **reserved field** `_sync_version` (string, HLC) to track the latest sync version for each document. This field is managed solely by the sync engine and should not be modified by application code.
|
|
251
|
+
|
|
252
|
+
**FR7.4** The engine supports MongoDB transactions (requires replica set) for atomic push operations. If transactions are not available (standalone), the library can fall back to non‑transactional writes with best‑effort consistency (explicitly documented as not recommended for production).
|
|
253
|
+
|
|
254
|
+
**FR7.5** On startup, the library must verify/initialize the required indexes and collections. It should provide a CLI or helper function for setup (`fastapi-offline-sync init`).
|
|
255
|
+
|
|
256
|
+
---
|
|
257
|
+
|
|
258
|
+
### FR8 – Observability & Monitoring
|
|
259
|
+
|
|
260
|
+
**FR8.1** Expose a Prometheus metrics endpoint (via `prometheus_fastapi_instrumentator` dependency) with these metrics:
|
|
261
|
+
- `sync_push_total` (counter): number of push requests, labels: status (success/failure).
|
|
262
|
+
- `sync_push_changes_total` (counter): total changes pushed.
|
|
263
|
+
- `sync_conflict_total` (counter): number of conflicts detected, labels: collection.
|
|
264
|
+
- `sync_pull_total` (counter): pull requests.
|
|
265
|
+
- `sync_pull_changes_served` (counter): total changes returned to clients.
|
|
266
|
+
- `sync_full_resync_total` (counter): full resync requests.
|
|
267
|
+
- `sync_oplog_size` (gauge): estimated document count in oplog.
|
|
268
|
+
- `sync_websocket_connections` (gauge): active WebSocket connections.
|
|
269
|
+
|
|
270
|
+
**FR8.2** Logging: important events (conflicts, full resync triggers, connection errors) are logged at WARNING or INFO level using standard Python logging with structured fields for easy parsing.
|
|
271
|
+
|
|
272
|
+
---
|
|
273
|
+
|
|
274
|
+
## 5. Non‑Functional Requirements
|
|
275
|
+
|
|
276
|
+
### NFR1 – Performance
|
|
277
|
+
- Push endpoint processing time per change: < 10 ms (excluding network). Batch of 50 changes: < 500 ms total.
|
|
278
|
+
- Pull endpoint response time: < 50 ms for batches up to 500 changes.
|
|
279
|
+
- Full resync: streaming begins within 100 ms; supports collections with 100k+ documents without timeouts.
|
|
280
|
+
- Oplog TTL index operates efficiently in aggregate.
|
|
281
|
+
|
|
282
|
+
### NFR2 – Reliability
|
|
283
|
+
- The sync engine must never lose a successfully pushed client change.
|
|
284
|
+
- In the event of a server crash during a push transaction, MongoDB’s transaction guarantees ensure either full application or full rollback.
|
|
285
|
+
- The oplog must be immutable (only appended to, never updated) to ensure pull consistency.
|
|
286
|
+
- Full resync must always deliver the latest committed state of the database, not an intermediate dirty read.
|
|
287
|
+
|
|
288
|
+
### NFR3 – Scalability
|
|
289
|
+
- Operates as a stateless FastAPI service (except for the WebSocket state, which can be handled via sticky sessions or pub/sub backplane in future).
|
|
290
|
+
- Can be horizontally scaled behind a load balancer; the use of MongoDB transactions and Change Streams ensures consistency across instances.
|
|
291
|
+
- WebSocket connections scale linearly; recommended to use `redis` pub/sub for broadcasting changes across multiple server instances (out of scope for initial release but architecture should not preclude it).
|
|
292
|
+
|
|
293
|
+
### NFR4 – Security
|
|
294
|
+
- JWT validation on every endpoint; support for token refresh implied (client handles reconnection).
|
|
295
|
+
- Input validation: `since` must be a valid HLC string; `limit` capped to 1000; collection names must pass a whitelist check if configured.
|
|
296
|
+
- No sensitive data in logs.
|
|
297
|
+
|
|
298
|
+
### NFR5 – Compatibility
|
|
299
|
+
- Python 3.9+
|
|
300
|
+
- FastAPI ≥ 0.100
|
|
301
|
+
- MongoDB ≥ 4.4 (replica set for transactions and Change Streams)
|
|
302
|
+
- `motor` (async MongoDB driver) or `pymongo` (synchronous fallback for certain operations if desired).
|
|
303
|
+
|
|
304
|
+
### NFR6 – Developer Experience
|
|
305
|
+
- Install via `pip install fastapi-offline-sync`.
|
|
306
|
+
- Mount the sync router:
|
|
307
|
+
```python
|
|
308
|
+
from fastapi import FastAPI
|
|
309
|
+
from fastapi_offline_sync import SyncRouter, SyncConfig
|
|
310
|
+
|
|
311
|
+
app = FastAPI()
|
|
312
|
+
sync_router = SyncRouter(config=SyncConfig(
|
|
313
|
+
mongodb_uri="mongodb://...",
|
|
314
|
+
database_name="mydb",
|
|
315
|
+
jwt_dependency=get_current_user # FastAPI dependency
|
|
316
|
+
))
|
|
317
|
+
app.include_router(sync_router)
|
|
318
|
+
```
|
|
319
|
+
- Comprehensive API documentation generated from OpenAPI.
|
|
320
|
+
- Configuration through environment variables or pydantic `Settings` class.
|
|
321
|
+
|
|
322
|
+
---
|
|
323
|
+
|
|
324
|
+
## 6. API Contract Details
|
|
325
|
+
|
|
326
|
+
### 6.1 Push
|
|
327
|
+
```
|
|
328
|
+
POST /sync/push
|
|
329
|
+
Authorization: Bearer <jwt>
|
|
330
|
+
Content-Type: application/json
|
|
331
|
+
|
|
332
|
+
{
|
|
333
|
+
"client_id": "device-xyz",
|
|
334
|
+
"changes": [
|
|
335
|
+
{
|
|
336
|
+
"collection": "tasks",
|
|
337
|
+
"operation": "upsert",
|
|
338
|
+
"doc_id": "task1",
|
|
339
|
+
"data": {"title": "Buy milk", "done": false},
|
|
340
|
+
"parent_version": "20260501T120000.000Z-0003"
|
|
341
|
+
}
|
|
342
|
+
]
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
HTTP 200
|
|
346
|
+
{
|
|
347
|
+
"results": [
|
|
348
|
+
{
|
|
349
|
+
"doc_id": "task1",
|
|
350
|
+
"status": "accepted",
|
|
351
|
+
"new_version": "20260502T150000.000Z-0005"
|
|
352
|
+
}
|
|
353
|
+
]
|
|
354
|
+
}
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
### 6.2 Pull
|
|
358
|
+
```
|
|
359
|
+
GET /sync/pull?since=20260502T150000.000Z-0005&collections=tasks&limit=100
|
|
360
|
+
HTTP 200
|
|
361
|
+
{
|
|
362
|
+
"changes": [...],
|
|
363
|
+
"last_seq": "20260502T151000.000Z-0008"
|
|
364
|
+
}
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
### 6.3 Full Resync
|
|
368
|
+
```
|
|
369
|
+
GET /sync/full?collections=tasks,projects&cursor=optional
|
|
370
|
+
Response: NDJSON stream with gzip
|
|
371
|
+
...
|
|
372
|
+
X-Sync-LastSeq: 20260502T151000.000Z-0008
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
### 6.4 WebSocket
|
|
376
|
+
```
|
|
377
|
+
ws://server/sync/stream?token=<jwt>
|
|
378
|
+
→ {"type":"subscribe","since":"...","collections":["tasks"]}
|
|
379
|
+
← {"version":"...","collection":"tasks","doc_id":"...","operation":"upsert","data":{...}}
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
---
|
|
383
|
+
|
|
384
|
+
## 7. Implementation Milestones (Server Only)
|
|
385
|
+
|
|
386
|
+
| Phase | Duration | Deliverables |
|
|
387
|
+
|-------|----------|--------------|
|
|
388
|
+
| **M1 – Setup & Core Libraries** | 1 week | Project scaffolding, HLC utility, MongoDB connection management, Pydantic models for API. |
|
|
389
|
+
| **M2 – Oplog & Push Endpoint** | 2 weeks | `POST /sync/push` with transaction support, oplog collection and indexes, conflict resolution (LWW). |
|
|
390
|
+
| **M3 – Pull & Full Resync** | 2 weeks | `GET /sync/pull`, `GET /sync/full`, full resync logic and stale detection. |
|
|
391
|
+
| **M4 – WebSocket & Real‑Time** | 2 weeks | `WS /sync/stream`, Change Streams integration, heartbeat. |
|
|
392
|
+
| **M5 – Hardening & Auth** | 1 week | Authentication dependency, authorization callbacks, per‑collection conflict policies, comprehensive error handling. |
|
|
393
|
+
| **M6 – Observability & Docs** | 1 week | Prometheus metrics, structured logging, OpenAPI docs, README, contribution guide. |
|
|
394
|
+
| **M7 – Testing & Release** | 1 week | Unit/integration tests (90%+ coverage), performance benchmarks, PyPI publication. |
|
|
395
|
+
|
|
396
|
+
---
|
|
397
|
+
|
|
398
|
+
## 8. Open Source & Community
|
|
399
|
+
|
|
400
|
+
- **License**: MIT.
|
|
401
|
+
- **Repository**: GitHub under `fisco/fastapi-offline-sync`.
|
|
402
|
+
- **Contributing**: issue templates, pre‑commit hooks (black, ruff, mypy), CI with GitHub Actions against multiple MongoDB versions.
|
|
403
|
+
- **Documentation site**: hosted on GitHub Pages with MkDocs.
|
|
404
|
+
- **Community channel**: Discord server.
|
|
405
|
+
|
|
406
|
+
---
|
|
407
|
+
|
|
408
|
+
This server‑side PRD defines exactly what `fastapi-offline-sync` must deliver to become the backbone of Fisco’s offline capabilities and a valuable open‑source asset. Would you like me to now draft the **client‑side PRD** or dive deeper into any specific server feature like the hybrid logical clock implementation?
|