sync-state-bridge 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sync_state/__init__.py +15 -0
- sync_state/client_socket.py +75 -0
- sync_state/errors.py +11 -0
- sync_state/presets.py +32 -0
- sync_state/qos.py +33 -0
- sync_state/sync_core.py +266 -0
- sync_state/transports/__init__.py +3 -0
- sync_state/transports/serial_stub.py +24 -0
- sync_state/transports/socket_adapter.py +101 -0
- sync_state_bridge-1.0.0.dist-info/METADATA +159 -0
- sync_state_bridge-1.0.0.dist-info/RECORD +14 -0
- sync_state_bridge-1.0.0.dist-info/WHEEL +5 -0
- sync_state_bridge-1.0.0.dist-info/licenses/LICENSE.md +21 -0
- sync_state_bridge-1.0.0.dist-info/top_level.txt +1 -0
sync_state/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from .sync_core import StateSync, StateSyncError, ProviderValidationError
|
|
2
|
+
from .qos import QoS, DropPolicy, TypeMetadata
|
|
3
|
+
from .presets import Presets
|
|
4
|
+
from .client_socket import StateSyncSocketClient
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"StateSync",
|
|
8
|
+
"StateSyncError",
|
|
9
|
+
"ProviderValidationError",
|
|
10
|
+
"QoS",
|
|
11
|
+
"DropPolicy",
|
|
12
|
+
"TypeMetadata",
|
|
13
|
+
"Presets",
|
|
14
|
+
"StateSyncSocketClient",
|
|
15
|
+
]
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import struct
|
|
4
|
+
import logging
|
|
5
|
+
from typing import Dict, Any, Callable, Optional
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger("sync_state_bridge.client")
|
|
8
|
+
LENGTH_PREFIX = "!I"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class StateSyncSocketClient:
|
|
12
|
+
"""Production socket client with automatic exponential backoff reconnection."""
|
|
13
|
+
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
host: str = "127.0.0.1",
|
|
17
|
+
port: int = 8765,
|
|
18
|
+
on_delta_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
|
|
19
|
+
max_reconnect_delay: float = 32.0
|
|
20
|
+
):
|
|
21
|
+
self.host = host
|
|
22
|
+
self.port = port
|
|
23
|
+
self.on_delta_callback = on_delta_callback
|
|
24
|
+
self.max_reconnect_delay = max_reconnect_delay
|
|
25
|
+
self.local_versions: Dict[str, int] = {}
|
|
26
|
+
self._running = False
|
|
27
|
+
|
|
28
|
+
async def connect_and_listen(self):
|
|
29
|
+
"""Persistent event-loop wrapper for network auto-healing."""
|
|
30
|
+
self._running = True
|
|
31
|
+
reconnect_delay = 1.0
|
|
32
|
+
|
|
33
|
+
while self._running:
|
|
34
|
+
try:
|
|
35
|
+
logger.info(f"Connecting to state sync server at {self.host}:{self.port}...")
|
|
36
|
+
reader, writer = await asyncio.open_connection(self.host, self.port)
|
|
37
|
+
reconnect_delay = 1.0 # Reset backoff interval on successful handshake
|
|
38
|
+
|
|
39
|
+
handshake_payload = json.dumps(self.local_versions).encode("utf-8")
|
|
40
|
+
header = struct.pack(LENGTH_PREFIX, len(handshake_payload))
|
|
41
|
+
writer.write(header)
|
|
42
|
+
writer.write(handshake_payload)
|
|
43
|
+
await writer.drain()
|
|
44
|
+
|
|
45
|
+
while self._running:
|
|
46
|
+
header_bytes = await reader.readexactly(4)
|
|
47
|
+
(length,) = struct.unpack(LENGTH_PREFIX, header_bytes)
|
|
48
|
+
payload_bytes = await reader.readexactly(length)
|
|
49
|
+
packet = json.loads(payload_bytes.decode("utf-8"))
|
|
50
|
+
|
|
51
|
+
event = packet.get("event")
|
|
52
|
+
data = packet.get("data", {})
|
|
53
|
+
|
|
54
|
+
if event == "manifest":
|
|
55
|
+
logger.info(f"Received server manifest. Schema version: {data.get('schema_version')}")
|
|
56
|
+
|
|
57
|
+
elif event == "delta":
|
|
58
|
+
type_name = data.get("type")
|
|
59
|
+
version = data.get("version", 0)
|
|
60
|
+
if type_name:
|
|
61
|
+
self.local_versions[type_name] = max(self.local_versions.get(type_name, 0), version)
|
|
62
|
+
|
|
63
|
+
if self.on_delta_callback:
|
|
64
|
+
if asyncio.iscoroutinefunction(self.on_delta_callback):
|
|
65
|
+
await self.on_delta_callback(data)
|
|
66
|
+
else:
|
|
67
|
+
self.on_delta_callback(data)
|
|
68
|
+
|
|
69
|
+
except (ConnectionRefusedError, asyncio.IncompleteReadError, BrokenPipeError, OSError) as e:
|
|
70
|
+
logger.warning(f"Connection error ({e}). Reconnecting in {reconnect_delay:.1f}s...")
|
|
71
|
+
await asyncio.sleep(reconnect_delay)
|
|
72
|
+
reconnect_delay = min(reconnect_delay * 2.0, self.max_reconnect_delay)
|
|
73
|
+
|
|
74
|
+
def stop(self):
|
|
75
|
+
self._running = False
|
sync_state/errors.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
class StateSyncError(Exception):
|
|
2
|
+
"""Base exception for StateSync operations."""
|
|
3
|
+
|
|
4
|
+
class ProviderValidationError(StateSyncError):
|
|
5
|
+
"""Raised when a snapshot provider returns invalid or unsafe data."""
|
|
6
|
+
|
|
7
|
+
class VersionMismatchError(StateSyncError):
|
|
8
|
+
"""Raised when client version is ahead of server (should not happen)."""
|
|
9
|
+
|
|
10
|
+
class SchemaMismatchError(StateSyncError):
|
|
11
|
+
"""Raised when schema versions differ."""
|
sync_state/presets.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from .qos import QoS, DropPolicy
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Presets:
|
|
5
|
+
"""Pre-configured QoS profiles for common network deployment scenarios."""
|
|
6
|
+
|
|
7
|
+
@staticmethod
|
|
8
|
+
def conservative() -> QoS:
|
|
9
|
+
"""Safe profile for general IP networking; ensures minimal delta loss."""
|
|
10
|
+
return QoS(
|
|
11
|
+
drop_policy=DropPolicy.CRITICAL,
|
|
12
|
+
ideal_cadence_ms=100,
|
|
13
|
+
ttl_ms=None
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
@staticmethod
|
|
17
|
+
def low_bandwidth() -> QoS:
|
|
18
|
+
"""Optimized for low-rate interfaces like serial, BLE, or satellite links."""
|
|
19
|
+
return QoS(
|
|
20
|
+
drop_policy=DropPolicy.CONFLATABLE,
|
|
21
|
+
ideal_cadence_ms=250,
|
|
22
|
+
ttl_ms=500
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def high_throughput() -> QoS:
|
|
27
|
+
"""High-frequency telemetry stream with aggressive delta drop allowance."""
|
|
28
|
+
return QoS(
|
|
29
|
+
drop_policy=DropPolicy.BEST_EFFORT,
|
|
30
|
+
ideal_cadence_ms=20,
|
|
31
|
+
ttl_ms=100
|
|
32
|
+
)
|
sync_state/qos.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DropPolicy(Enum):
|
|
8
|
+
"""Defines how deltas are handled when transport buffers overflow."""
|
|
9
|
+
CRITICAL = "never_drop" # Guarded: Queued until delivered, never discarded
|
|
10
|
+
CONFLATABLE = "keep_latest" # Conflated: Drops intermediate deltas in favor of newest
|
|
11
|
+
BEST_EFFORT = "drop_on_overflow" # Discardable: Dropped immediately under queue pressure
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class QoS:
|
|
16
|
+
"""Quality of Service metadata for entity streams."""
|
|
17
|
+
drop_policy: DropPolicy = DropPolicy.CONFLATABLE
|
|
18
|
+
ideal_cadence_ms: int = 50 # Target refresh interval in ms
|
|
19
|
+
ttl_ms: Optional[int] = 1000 # Time-To-Live in queue before delta expires
|
|
20
|
+
|
|
21
|
+
def is_expired(self, created_at_timestamp: float) -> bool:
|
|
22
|
+
"""Check if a delta has exceeded its TTL in transport queue."""
|
|
23
|
+
if self.ttl_ms is None or self.drop_policy == DropPolicy.CRITICAL:
|
|
24
|
+
return False
|
|
25
|
+
return (time.time() - created_at_timestamp) * 1000.0 > self.ttl_ms
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class TypeMetadata:
|
|
30
|
+
"""Registration metadata for snapshot providers."""
|
|
31
|
+
type_name: str
|
|
32
|
+
qos: QoS = QoS()
|
|
33
|
+
max_frame_bytes: int = 1_048_576 # 1MB default chunking threshold
|
sync_state/sync_core.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import hashlib
|
|
3
|
+
import asyncio
|
|
4
|
+
import copy
|
|
5
|
+
import time
|
|
6
|
+
from collections import defaultdict, deque
|
|
7
|
+
from typing import Dict, List, Any, Callable, Optional, Set, AsyncIterator
|
|
8
|
+
from .qos import QoS, DropPolicy, TypeMetadata # if you have these
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def canonical_hash(entity: Dict[str, Any]) -> str:
|
|
12
|
+
"""Deterministic, canonical SHA-256 hash calculation for dictionaries."""
|
|
13
|
+
if "_v" in entity:
|
|
14
|
+
return str(entity["_v"])
|
|
15
|
+
|
|
16
|
+
def _default_encoder(o: Any) -> Any:
|
|
17
|
+
if isinstance(o, (set, tuple)):
|
|
18
|
+
return list(o)
|
|
19
|
+
if hasattr(o, "to_dict") and callable(o.to_dict):
|
|
20
|
+
return o.to_dict()
|
|
21
|
+
return sorted(list(o.__dict__.items())) if hasattr(o, "__dict__") else str(o)
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import orjson
|
|
25
|
+
encoded = orjson.dumps(entity, option=orjson.OPT_SORT_KEYS, default=_default_encoder)
|
|
26
|
+
except ImportError:
|
|
27
|
+
encoded = json.dumps(entity, sort_keys=True, default=_default_encoder).encode('utf-8')
|
|
28
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
29
|
+
|
|
30
|
+
class StateSyncError(Exception):
|
|
31
|
+
"""Base exception for StateSync operations."""
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ProviderValidationError(StateSyncError):
|
|
36
|
+
"""Raised when a snapshot provider returns invalid or unsafe data."""
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class StateSync:
|
|
41
|
+
def __init__(self, id_key: str = "id", max_history: int = 1000, schema_version: int = 1):
|
|
42
|
+
self.id_key = id_key
|
|
43
|
+
self.max_history = max_history
|
|
44
|
+
self.schema_version = schema_version
|
|
45
|
+
|
|
46
|
+
self._lock = asyncio.Lock()
|
|
47
|
+
self._dirty_types: Set[str] = set()
|
|
48
|
+
self._snapshot_providers: Dict[str, Callable[[], List[Dict]]] = {}
|
|
49
|
+
|
|
50
|
+
self._versions: Dict[str, int] = defaultdict(int)
|
|
51
|
+
self._change_logs: Dict[str, deque] = defaultdict(lambda: deque(maxlen=max_history))
|
|
52
|
+
self._current_snapshots: Dict[str, Dict[Any, Dict]] = {}
|
|
53
|
+
self._type_metadata: Dict[str, TypeMetadata] = {} # <-- add this
|
|
54
|
+
|
|
55
|
+
self._stream_events: List[asyncio.Event] = []
|
|
56
|
+
|
|
57
|
+
def register_snapshot_provider(
|
|
58
|
+
self,
|
|
59
|
+
type_name: str,
|
|
60
|
+
provider: Callable[[], List[Dict]],
|
|
61
|
+
qos: Optional[QoS] = None,
|
|
62
|
+
max_frame_bytes: int = 1_048_576
|
|
63
|
+
):
|
|
64
|
+
from .qos import QoS, TypeMetadata # ensure import is available
|
|
65
|
+
qos_obj = qos or QoS()
|
|
66
|
+
self._snapshot_providers[type_name] = provider
|
|
67
|
+
self._type_metadata[type_name] = TypeMetadata(
|
|
68
|
+
type_name=type_name,
|
|
69
|
+
qos=qos_obj,
|
|
70
|
+
max_frame_bytes=max_frame_bytes
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def mark_dirty(self, type_name: str):
|
|
74
|
+
self._dirty_types.add(type_name)
|
|
75
|
+
|
|
76
|
+
def _canonical_hash(self, entity: Dict) -> str:
|
|
77
|
+
if "_v" in entity:
|
|
78
|
+
return str(entity["_v"])
|
|
79
|
+
|
|
80
|
+
def _default_encoder(o):
|
|
81
|
+
if isinstance(o, (set, tuple)):
|
|
82
|
+
return list(o)
|
|
83
|
+
return f"__repr__:{type(o).__name__}:{repr(o)}"
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
import orjson
|
|
87
|
+
encoded = orjson.dumps(entity, option=orjson.OPT_SORT_KEYS, default=_default_encoder)
|
|
88
|
+
except ImportError:
|
|
89
|
+
encoded = json.dumps(entity, sort_keys=True, default=_default_encoder).encode('utf-8')
|
|
90
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
91
|
+
|
|
92
|
+
def _validate_and_sanitize_snapshot(self, raw_entities: Any, type_name: str) -> Dict[Any, Dict]:
|
|
93
|
+
if not isinstance(raw_entities, (list, tuple)):
|
|
94
|
+
raise ProviderValidationError(f"Provider for '{type_name}' must return a list or tuple.")
|
|
95
|
+
|
|
96
|
+
sanitized = {}
|
|
97
|
+
for idx, entity in enumerate(raw_entities):
|
|
98
|
+
if not isinstance(entity, dict):
|
|
99
|
+
raise ProviderValidationError(f"Entity at index {idx} in '{type_name}' is not a dict.")
|
|
100
|
+
if self.id_key not in entity:
|
|
101
|
+
raise ProviderValidationError(f"Entity at index {idx} in '{type_name}' missing key '{self.id_key}'.")
|
|
102
|
+
eid = entity[self.id_key]
|
|
103
|
+
if eid is None or not isinstance(eid, (str, int)):
|
|
104
|
+
raise ProviderValidationError(f"Entity ID at index {idx} in '{type_name}' must be str or int.")
|
|
105
|
+
if eid in sanitized:
|
|
106
|
+
raise ProviderValidationError(f"Duplicate entity ID '{eid}' detected in '{type_name}'.")
|
|
107
|
+
sanitized[eid] = copy.deepcopy(entity)
|
|
108
|
+
return sanitized
|
|
109
|
+
|
|
110
|
+
async def commit(self):
|
|
111
|
+
dirty_types_to_process = []
|
|
112
|
+
async with self._lock:
|
|
113
|
+
if not self._dirty_types:
|
|
114
|
+
return
|
|
115
|
+
dirty_types_to_process = list(self._dirty_types)
|
|
116
|
+
self._dirty_types.clear()
|
|
117
|
+
|
|
118
|
+
fetched_snapshots: Dict[str, Dict[Any, Dict]] = {}
|
|
119
|
+
for type_name in dirty_types_to_process:
|
|
120
|
+
if type_name in self._snapshot_providers:
|
|
121
|
+
try:
|
|
122
|
+
raw_data = self._snapshot_providers[type_name]()
|
|
123
|
+
fetched_snapshots[type_name] = self._validate_and_sanitize_snapshot(raw_data, type_name)
|
|
124
|
+
except Exception:
|
|
125
|
+
async with self._lock:
|
|
126
|
+
self._dirty_types.add(type_name)
|
|
127
|
+
raise
|
|
128
|
+
|
|
129
|
+
async with self._lock:
|
|
130
|
+
for type_name, new_snapshot in fetched_snapshots.items():
|
|
131
|
+
old_snapshot = self._current_snapshots.get(type_name, {})
|
|
132
|
+
old_ids = set(old_snapshot.keys())
|
|
133
|
+
new_ids = set(new_snapshot.keys())
|
|
134
|
+
|
|
135
|
+
self._versions[type_name] += 1
|
|
136
|
+
current_version = self._versions[type_name]
|
|
137
|
+
|
|
138
|
+
added = new_ids - old_ids
|
|
139
|
+
deleted = old_ids - new_ids
|
|
140
|
+
common = new_ids & old_ids
|
|
141
|
+
|
|
142
|
+
updated = [
|
|
143
|
+
eid for eid in common
|
|
144
|
+
if self._canonical_hash(old_snapshot[eid]) != self._canonical_hash(new_snapshot[eid])
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
self._change_logs[type_name].append({
|
|
148
|
+
"version": current_version,
|
|
149
|
+
"added": list(added),
|
|
150
|
+
"updated": updated,
|
|
151
|
+
"deleted": list(deleted)
|
|
152
|
+
})
|
|
153
|
+
self._current_snapshots[type_name] = new_snapshot
|
|
154
|
+
|
|
155
|
+
# Notify all active streams
|
|
156
|
+
for evt in self._stream_events:
|
|
157
|
+
evt.set()
|
|
158
|
+
|
|
159
|
+
def get_delta(self, type_name: str, client_version: int) -> Dict[str, Any]:
|
|
160
|
+
snapshot = self._current_snapshots.get(type_name, {})
|
|
161
|
+
history = self._change_logs.get(type_name)
|
|
162
|
+
current_version = self._versions.get(type_name, 0)
|
|
163
|
+
|
|
164
|
+
if not history or client_version < history[0]["version"] or client_version > current_version:
|
|
165
|
+
return {
|
|
166
|
+
"type": type_name,
|
|
167
|
+
"full": True,
|
|
168
|
+
"version": current_version,
|
|
169
|
+
"added": list(snapshot.values()),
|
|
170
|
+
"updated": [],
|
|
171
|
+
"deleted": []
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
entity_state: Dict[Any, str] = {}
|
|
175
|
+
for entry in history:
|
|
176
|
+
if entry["version"] <= client_version:
|
|
177
|
+
continue
|
|
178
|
+
for eid in entry["added"]:
|
|
179
|
+
entity_state[eid] = "added"
|
|
180
|
+
for eid in entry["updated"]:
|
|
181
|
+
if eid not in entity_state:
|
|
182
|
+
entity_state[eid] = "updated"
|
|
183
|
+
elif entity_state[eid] == "added":
|
|
184
|
+
continue
|
|
185
|
+
elif entity_state[eid] == "deleted":
|
|
186
|
+
continue
|
|
187
|
+
for eid in entry["deleted"]:
|
|
188
|
+
if entity_state.get(eid) == "added":
|
|
189
|
+
del entity_state[eid]
|
|
190
|
+
else:
|
|
191
|
+
entity_state[eid] = "deleted"
|
|
192
|
+
|
|
193
|
+
added_ids = [eid for eid, state in entity_state.items() if state == "added"]
|
|
194
|
+
updated_ids = [eid for eid, state in entity_state.items() if state == "updated"]
|
|
195
|
+
deleted_ids = [eid for eid, state in entity_state.items() if state == "deleted"]
|
|
196
|
+
|
|
197
|
+
return {
|
|
198
|
+
"type": type_name,
|
|
199
|
+
"full": False,
|
|
200
|
+
"version": current_version,
|
|
201
|
+
"added": [snapshot[eid] for eid in added_ids if eid in snapshot],
|
|
202
|
+
"updated": [snapshot[eid] for eid in updated_ids if eid in snapshot],
|
|
203
|
+
"deleted": deleted_ids
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async def stream_deltas(self, client_versions: Dict[str, int], keepalive_interval: int = 15):
|
|
207
|
+
# Create a per‑stream event on the current loop
|
|
208
|
+
stream_event = asyncio.Event()
|
|
209
|
+
self._stream_events.append(stream_event)
|
|
210
|
+
|
|
211
|
+
try:
|
|
212
|
+
local_client_versions = dict(client_versions)
|
|
213
|
+
|
|
214
|
+
# Initial sync
|
|
215
|
+
deltas_to_send = []
|
|
216
|
+
full_manifest_versions = {}
|
|
217
|
+
|
|
218
|
+
async with self._lock:
|
|
219
|
+
for type_name in self._snapshot_providers:
|
|
220
|
+
c_ver = local_client_versions.get(type_name, 0)
|
|
221
|
+
delta = self.get_delta(type_name, c_ver)
|
|
222
|
+
full_manifest_versions[type_name] = self._versions.get(type_name, 0)
|
|
223
|
+
if delta["full"] or delta["added"] or delta["updated"] or delta["deleted"]:
|
|
224
|
+
deltas_to_send.append(delta)
|
|
225
|
+
local_client_versions[type_name] = delta["version"]
|
|
226
|
+
|
|
227
|
+
manifest = {
|
|
228
|
+
"schema_version": self.schema_version,
|
|
229
|
+
"versions": full_manifest_versions,
|
|
230
|
+
"types": list(self._snapshot_providers.keys())
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
yield f"event: manifest\ndata: {json.dumps(manifest)}\n\n"
|
|
234
|
+
|
|
235
|
+
for delta in deltas_to_send:
|
|
236
|
+
yield f"event: delta\ndata: {json.dumps(delta)}\n\n"
|
|
237
|
+
|
|
238
|
+
# Reactive loop
|
|
239
|
+
while True:
|
|
240
|
+
try:
|
|
241
|
+
await asyncio.wait_for(stream_event.wait(), timeout=keepalive_interval)
|
|
242
|
+
except asyncio.TimeoutError:
|
|
243
|
+
# Keepalive
|
|
244
|
+
yield f"event: keepalive\ndata: {{}}\n\n"
|
|
245
|
+
continue
|
|
246
|
+
|
|
247
|
+
# Clear the event so it can be used again
|
|
248
|
+
stream_event.clear()
|
|
249
|
+
|
|
250
|
+
# A commit happened – check for new deltas
|
|
251
|
+
deltas_to_send = []
|
|
252
|
+
async with self._lock:
|
|
253
|
+
for type_name in self._snapshot_providers:
|
|
254
|
+
c_ver = local_client_versions.get(type_name, 0)
|
|
255
|
+
delta = self.get_delta(type_name, c_ver)
|
|
256
|
+
if delta["full"] or delta["added"] or delta["updated"] or delta["deleted"]:
|
|
257
|
+
deltas_to_send.append(delta)
|
|
258
|
+
local_client_versions[type_name] = delta["version"]
|
|
259
|
+
|
|
260
|
+
for delta in deltas_to_send:
|
|
261
|
+
yield f"event: delta\ndata: {json.dumps(delta)}\n\n"
|
|
262
|
+
|
|
263
|
+
finally:
|
|
264
|
+
# Remove this stream's event from the list
|
|
265
|
+
if stream_event in self._stream_events:
|
|
266
|
+
self._stream_events.remove(stream_event)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from typing import Any, Dict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SerialTransportAdapter:
|
|
5
|
+
def __init__(self, port: str, baudrate: int = 115200, max_queue_size: int = 25):
|
|
6
|
+
self.port = port
|
|
7
|
+
self.baudrate = baudrate
|
|
8
|
+
self.max_queue_size = max_queue_size
|
|
9
|
+
raise NotImplementedError("SerialTransportAdapter is currently a stub. Implementation planned for v0.2.0.")
|
|
10
|
+
|
|
11
|
+
async def start(self) -> None:
|
|
12
|
+
raise NotImplementedError
|
|
13
|
+
|
|
14
|
+
async def send_delta(self, delta: Dict[str, Any]) -> None:
|
|
15
|
+
# Lazy import to avoid hard dependency at module load
|
|
16
|
+
try:
|
|
17
|
+
import cobs # noqa: F401
|
|
18
|
+
import serial # noqa: F401
|
|
19
|
+
except ImportError as e:
|
|
20
|
+
raise RuntimeError(
|
|
21
|
+
"Serial transport dependencies missing. "
|
|
22
|
+
"Install via 'pip install sync-state-bridge[serial]'"
|
|
23
|
+
) from e
|
|
24
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import struct
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger("sync_state_bridge.socket")
|
|
7
|
+
LENGTH_PREFIX = "!I"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class StateSyncSocketServer:
|
|
11
|
+
def __init__(self, state_sync, max_queue_size: int = 100):
|
|
12
|
+
self.state_sync = state_sync
|
|
13
|
+
self.max_queue_size = max_queue_size
|
|
14
|
+
self.metrics = {
|
|
15
|
+
"queue_length": 0,
|
|
16
|
+
"deltas_dropped": 0,
|
|
17
|
+
"full_snapshots_sent": 0,
|
|
18
|
+
"client_congested_count": 0
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async def handle_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
|
|
22
|
+
outbound_queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=self.max_queue_size)
|
|
23
|
+
priority_snapshot_queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=10)
|
|
24
|
+
is_congested = False
|
|
25
|
+
|
|
26
|
+
async def _write_worker():
|
|
27
|
+
nonlocal is_congested
|
|
28
|
+
while True:
|
|
29
|
+
if not priority_snapshot_queue.empty():
|
|
30
|
+
packet = await priority_snapshot_queue.get()
|
|
31
|
+
priority_snapshot_queue.task_done()
|
|
32
|
+
else:
|
|
33
|
+
packet = await outbound_queue.get()
|
|
34
|
+
outbound_queue.task_done()
|
|
35
|
+
|
|
36
|
+
self.metrics["queue_length"] = outbound_queue.qsize()
|
|
37
|
+
payload = json.dumps(packet).encode("utf-8")
|
|
38
|
+
header = struct.pack(LENGTH_PREFIX, len(payload))
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
writer.write(header)
|
|
42
|
+
writer.write(payload)
|
|
43
|
+
await writer.drain()
|
|
44
|
+
except (ConnectionResetError, BrokenPipeError, asyncio.CancelledError):
|
|
45
|
+
break
|
|
46
|
+
|
|
47
|
+
write_task = asyncio.create_task(_write_worker())
|
|
48
|
+
packet_gen = None
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
header_bytes = await reader.readexactly(4)
|
|
52
|
+
(length,) = struct.unpack(LENGTH_PREFIX, header_bytes)
|
|
53
|
+
client_versions = json.loads((await reader.readexactly(length)).decode("utf-8"))
|
|
54
|
+
|
|
55
|
+
packet_gen = self.state_sync.iter_packets(client_versions)
|
|
56
|
+
async for packet in packet_gen:
|
|
57
|
+
event_type = packet.get("event")
|
|
58
|
+
data = packet.get("data", {})
|
|
59
|
+
|
|
60
|
+
if event_type == "delta" and data.get("full"):
|
|
61
|
+
self.metrics["full_snapshots_sent"] += 1
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
outbound_queue.put_nowait(packet)
|
|
65
|
+
except asyncio.QueueFull:
|
|
66
|
+
self.metrics["deltas_dropped"] += 1
|
|
67
|
+
if not is_congested:
|
|
68
|
+
is_congested = True
|
|
69
|
+
self.metrics["client_congested_count"] += 1
|
|
70
|
+
logger.warning("Client transport congested. Dropping delta frame.")
|
|
71
|
+
|
|
72
|
+
while not outbound_queue.empty():
|
|
73
|
+
try:
|
|
74
|
+
outbound_queue.get_nowait()
|
|
75
|
+
outbound_queue.task_done()
|
|
76
|
+
except asyncio.QueueEmpty:
|
|
77
|
+
break
|
|
78
|
+
|
|
79
|
+
recovery_packet = {
|
|
80
|
+
"event": "delta",
|
|
81
|
+
"data": self.state_sync.get_delta(data.get("type", ""), client_version=-1)
|
|
82
|
+
}
|
|
83
|
+
try:
|
|
84
|
+
priority_snapshot_queue.put_nowait(recovery_packet)
|
|
85
|
+
except asyncio.QueueFull:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError):
|
|
89
|
+
pass
|
|
90
|
+
finally:
|
|
91
|
+
if packet_gen is not None:
|
|
92
|
+
await packet_gen.aclose()
|
|
93
|
+
write_task.cancel()
|
|
94
|
+
writer.close()
|
|
95
|
+
try:
|
|
96
|
+
await writer.wait_closed()
|
|
97
|
+
except Exception:
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
async def start_tcp(self, host: str = "127.0.0.1", port: int = 8765):
|
|
101
|
+
return await asyncio.start_server(self.handle_connection, host, port)
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sync-state-bridge
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Versioned delta-state synchronization engine for real-time systems, IPC, and embedded applications
|
|
5
|
+
Author-email: Andrew Kingdom <1809762+akingdom@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/akingdom/sync-state-bridge
|
|
8
|
+
Project-URL: Bug-Tracker, https://github.com/akingdom/sync-state-bridge/issues
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Requires-Python: >=3.7
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE.md
|
|
21
|
+
Provides-Extra: fast
|
|
22
|
+
Requires-Dist: orjson>=3.8.0; extra == "fast"
|
|
23
|
+
Provides-Extra: serial
|
|
24
|
+
Requires-Dist: pyserial>=3.5; extra == "serial"
|
|
25
|
+
Requires-Dist: cobs>=1.2.0; extra == "serial"
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
29
|
+
Requires-Dist: flake8>=6.0.0; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# sync-state-bridge
|
|
33
|
+
|
|
34
|
+
A deterministic, race‑safe state synchronisation bridge for real‑time applications, with built‑in QoS, backpressure handling, and network resilience.
|
|
35
|
+
|
|
36
|
+
- **Versioned, per‑type sync** – each entity type has its own version history.
|
|
37
|
+
- **Segmented deltas** – send only what changed, with a manifest first.
|
|
38
|
+
- **FSM lifecycle** – correct handling of add→update→delete sequences.
|
|
39
|
+
- **Resilient reconnection** – out‑of‑order drop, monotonicity guards, schema mismatch handling.
|
|
40
|
+
- **Quality of Service (QoS)** – define drop policies (CRITICAL, CONFLATABLE, BEST_EFFORT) per entity type.
|
|
41
|
+
- **Backpressure‑aware transports** – bounded queues, priority full‑snapshot injection, congestion metrics.
|
|
42
|
+
- **Reconnecting client** – exponential backoff, version tracking, automatic recovery.
|
|
43
|
+
- **Pure Python + JS** – no binary dependencies.
|
|
44
|
+
|
|
45
|
+
## Quick Start
|
|
46
|
+
|
|
47
|
+
Install the server package:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install sync-state-bridge
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
For optional features (faster JSON, serial transport):
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install sync-state-bridge[fast,serial]
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Basic Usage
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from sync_state import StateSync, Presets
|
|
63
|
+
|
|
64
|
+
sync = StateSync()
|
|
65
|
+
|
|
66
|
+
# Register a snapshot provider with a QoS profile
|
|
67
|
+
sync.register_snapshot_provider(
|
|
68
|
+
"vehicles",
|
|
69
|
+
get_vehicles,
|
|
70
|
+
qos=Presets.low_bandwidth() # optimised for slow links
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# After changes:
|
|
74
|
+
sync.mark_dirty("vehicles")
|
|
75
|
+
await sync.commit()
|
|
76
|
+
|
|
77
|
+
# Stream deltas over HTTP (SSE)
|
|
78
|
+
from fastapi import FastAPI, StreamingResponse
|
|
79
|
+
app = FastAPI()
|
|
80
|
+
|
|
81
|
+
@app.get("/stream")
|
|
82
|
+
async def stream(versions: str = "{}"):
|
|
83
|
+
return StreamingResponse(
|
|
84
|
+
sync.stream_deltas(json.loads(versions)),
|
|
85
|
+
media_type="text/event-stream"
|
|
86
|
+
)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Socket Server (for low‑level IPC)
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from sync_state.transports import StateSyncSocketServer
|
|
93
|
+
|
|
94
|
+
server = StateSyncSocketServer(sync)
|
|
95
|
+
await server.start_tcp(host="0.0.0.0", port=8765)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Reconnecting Client
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from sync_state import StateSyncSocketClient
|
|
102
|
+
|
|
103
|
+
def handle_delta(delta):
|
|
104
|
+
print(f"Update: {delta}")
|
|
105
|
+
|
|
106
|
+
client = StateSyncSocketClient(
|
|
107
|
+
host="127.0.0.1",
|
|
108
|
+
port=8765,
|
|
109
|
+
on_delta_callback=handle_delta
|
|
110
|
+
)
|
|
111
|
+
await client.connect_and_listen()
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Quality of Service (QoS)
|
|
115
|
+
|
|
116
|
+
Each entity type can have a `QoS` profile:
|
|
117
|
+
|
|
118
|
+
| Policy | Behaviour |
|
|
119
|
+
|---------------|-----------|
|
|
120
|
+
| `CRITICAL` | Never dropped; queued until delivered. |
|
|
121
|
+
| `CONFLATABLE` | Intermediate deltas dropped; only the latest is sent (ideal for high‑frequency telemetry). |
|
|
122
|
+
| `BEST_EFFORT` | Discarded immediately under queue pressure. |
|
|
123
|
+
|
|
124
|
+
Pre‑configured profiles (`Presets.conservative()`, `Presets.low_bandwidth()`, `Presets.high_throughput()`) are provided.
|
|
125
|
+
|
|
126
|
+
## Testing
|
|
127
|
+
|
|
128
|
+
Run the unit tests:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
pytest tests/
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The test suite covers:
|
|
135
|
+
- Deterministic hashing (`canonical_hash`)
|
|
136
|
+
- Commit & delta generation
|
|
137
|
+
- Version‑gap full‑snapshot recovery
|
|
138
|
+
|
|
139
|
+
## Demos
|
|
140
|
+
|
|
141
|
+
- **Chat** – real‑time message broadcast with shared history.
|
|
142
|
+
- **Game** – "Find the Clusters" demonstrating turn‑based sync.
|
|
143
|
+
|
|
144
|
+
Run the demos:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
cd examples/chat && python server.py
|
|
148
|
+
```
|
|
149
|
+
```bash
|
|
150
|
+
cd examples/game && python server.py
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Protocol
|
|
154
|
+
|
|
155
|
+
See [`PROTOCOL.md`](./PROTOCOL.md) for the full SSE‑based delta protocol.
|
|
156
|
+
|
|
157
|
+
## License
|
|
158
|
+
|
|
159
|
+
MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
sync_state/__init__.py,sha256=nD0VlMOUJKvJDEmBDas9jcbUof8eOaHCuWiE3G-JHCY,377
|
|
2
|
+
sync_state/client_socket.py,sha256=3HZa5CAPDAlahHnwHskOe0A1us3H4c2GvuXcgZAwyhk,3093
|
|
3
|
+
sync_state/errors.py,sha256=SgG4gnP3-E2jlJVr5Z-MJLP6CtmBaEfJUmu0bW4KpMc,417
|
|
4
|
+
sync_state/presets.py,sha256=ezJ46z0lfH5IYVtPhO7RWW5JgTJ6I5A68OVdRl18Vns,937
|
|
5
|
+
sync_state/qos.py,sha256=8XhpjOJUlQh_0vTu7yPSOUDiWgRoNHf6aO_e9QxLxck,1269
|
|
6
|
+
sync_state/sync_core.py,sha256=3DOUxDTmSGcksXYgzxGELk5tzhcc1m3pUymuEAnkJRs,10757
|
|
7
|
+
sync_state/transports/__init__.py,sha256=dxlxX41E6OXJMpVj8x6TbK_UZzFdFEKOz5wjZ10oEKc,87
|
|
8
|
+
sync_state/transports/serial_stub.py,sha256=xb2mLeFDyfyzwJHon3XmwXSpE0fkfi8coaR3T1ScbDY,900
|
|
9
|
+
sync_state/transports/socket_adapter.py,sha256=uChlw_6GoMer17TESYkUdB6xDyoC2hun6rCrqmJZJiE,3928
|
|
10
|
+
sync_state_bridge-1.0.0.dist-info/licenses/LICENSE.md,sha256=3PS64E9B4QMAlNM47CgxOtGQm3mEfvDIOmSde6rJObU,1071
|
|
11
|
+
sync_state_bridge-1.0.0.dist-info/METADATA,sha256=mWqIZ_xOcDVYV0KAfH5FlHxcenuFxirQN89h1OCkOOc,4555
|
|
12
|
+
sync_state_bridge-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
13
|
+
sync_state_bridge-1.0.0.dist-info/top_level.txt,sha256=XcYcbcBKi6kZhdY2507ekEjcKuhM1HP6B-XFmjbQEYY,11
|
|
14
|
+
sync_state_bridge-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Andrew Kingdom
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sync_state
|