edgesync 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edgesync/__init__.py +61 -0
- edgesync/client.py +227 -0
- edgesync/config.py +70 -0
- edgesync/exceptions.py +51 -0
- edgesync/logging.py +16 -0
- edgesync/models/__init__.py +16 -0
- edgesync/models/delivery.py +69 -0
- edgesync/models/message.py +62 -0
- edgesync/models/receipt.py +21 -0
- edgesync/models/stats.py +27 -0
- edgesync/py.typed +0 -0
- edgesync/queue/__init__.py +5 -0
- edgesync/queue/manager.py +92 -0
- edgesync/retry/__init__.py +5 -0
- edgesync/retry/backoff.py +42 -0
- edgesync/retry/policy.py +56 -0
- edgesync/storage/__init__.py +6 -0
- edgesync/storage/base.py +109 -0
- edgesync/storage/migrations.py +70 -0
- edgesync/storage/sqlite.py +529 -0
- edgesync/transports/__init__.py +7 -0
- edgesync/transports/base.py +30 -0
- edgesync/transports/http.py +112 -0
- edgesync/transports/registry.py +52 -0
- edgesync/utils/__init__.py +1 -0
- edgesync/utils/clock.py +50 -0
- edgesync/utils/ids.py +20 -0
- edgesync/worker/__init__.py +5 -0
- edgesync/worker/lifecycle.py +62 -0
- edgesync/worker/scheduler.py +20 -0
- edgesync/worker/sync_worker.py +162 -0
- edgesync-0.2.0.dist-info/METADATA +155 -0
- edgesync-0.2.0.dist-info/RECORD +35 -0
- edgesync-0.2.0.dist-info/WHEEL +4 -0
- edgesync-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Maps destination names to transports.
|
|
2
|
+
|
|
3
|
+
``EdgeSync.publish(data, destination="telemetry")`` is routed through this
|
|
4
|
+
registry to the transport registered under that name, falling back to the
|
|
5
|
+
default destination when none is specified.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
|
|
12
|
+
from edgesync.exceptions import ConfigurationError
|
|
13
|
+
from edgesync.transports.base import Transport
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TransportRegistry:
|
|
17
|
+
"""Owns the lifecycle of every registered :class:`Transport`."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, default_destination: str) -> None:
|
|
20
|
+
self._default_destination = default_destination
|
|
21
|
+
self._transports: dict[str, Transport] = {}
|
|
22
|
+
self._started = False
|
|
23
|
+
|
|
24
|
+
def register(self, name: str, transport: Transport) -> None:
|
|
25
|
+
self._transports[name] = transport
|
|
26
|
+
|
|
27
|
+
def get(self, destination: str | None) -> Transport:
|
|
28
|
+
name = destination or self._default_destination
|
|
29
|
+
try:
|
|
30
|
+
return self._transports[name]
|
|
31
|
+
except KeyError:
|
|
32
|
+
raise ConfigurationError(
|
|
33
|
+
f"no transport registered for destination {name!r} "
|
|
34
|
+
f"(registered: {sorted(self._transports)})"
|
|
35
|
+
) from None
|
|
36
|
+
|
|
37
|
+
def destinations(self) -> list[str]:
|
|
38
|
+
return list(self._transports)
|
|
39
|
+
|
|
40
|
+
async def start_all(self) -> None:
|
|
41
|
+
if self._started:
|
|
42
|
+
return
|
|
43
|
+
await asyncio.gather(*(t.start() for t in self._transports.values()))
|
|
44
|
+
self._started = True
|
|
45
|
+
|
|
46
|
+
async def close_all(self) -> None:
|
|
47
|
+
if not self._transports:
|
|
48
|
+
return
|
|
49
|
+
await asyncio.gather(
|
|
50
|
+
*(t.close() for t in self._transports.values()), return_exceptions=True
|
|
51
|
+
)
|
|
52
|
+
self._started = False
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Internal utility helpers (clock abstraction, ID generation)."""
|
edgesync/utils/clock.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Clock abstraction.
|
|
2
|
+
|
|
3
|
+
Business logic must never call ``datetime.now()`` or ``time.time()``
|
|
4
|
+
directly. Going through a :class:`Clock` keeps all internal timestamps
|
|
5
|
+
timezone-aware UTC and lets tests substitute a deterministic fake clock
|
|
6
|
+
instead of sleeping or racing against wall-clock time.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from typing import Protocol, runtime_checkable
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@runtime_checkable
|
|
16
|
+
class Clock(Protocol):
|
|
17
|
+
"""A source of the current time."""
|
|
18
|
+
|
|
19
|
+
def now(self) -> datetime:
|
|
20
|
+
"""Return the current time as a timezone-aware UTC datetime."""
|
|
21
|
+
...
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SystemClock:
|
|
25
|
+
"""Clock backed by the system wall clock."""
|
|
26
|
+
|
|
27
|
+
def now(self) -> datetime:
|
|
28
|
+
return datetime.now(timezone.utc)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class FixedClock:
|
|
32
|
+
"""Deterministic clock for tests.
|
|
33
|
+
|
|
34
|
+
The clock starts at ``start`` (or the current system time) and only
|
|
35
|
+
advances when :meth:`advance` is called explicitly.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, start: datetime | None = None) -> None:
|
|
39
|
+
self._current = start or SystemClock().now()
|
|
40
|
+
|
|
41
|
+
def now(self) -> datetime:
|
|
42
|
+
return self._current
|
|
43
|
+
|
|
44
|
+
def advance(self, seconds: float) -> None:
|
|
45
|
+
from datetime import timedelta
|
|
46
|
+
|
|
47
|
+
self._current = self._current + timedelta(seconds=seconds)
|
|
48
|
+
|
|
49
|
+
def set(self, when: datetime) -> None:
|
|
50
|
+
self._current = when
|
edgesync/utils/ids.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""ID generation helpers.
|
|
2
|
+
|
|
3
|
+
Message IDs are globally unique (UUID4) and are exposed to transports so
|
|
4
|
+
destinations can implement idempotent processing (see
|
|
5
|
+
``HTTPTransport``'s ``Idempotency-Key`` header).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import uuid
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def generate_message_id() -> str:
|
|
14
|
+
"""Generate a globally unique message ID."""
|
|
15
|
+
return str(uuid.uuid4())
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def generate_lease_id() -> str:
|
|
19
|
+
"""Generate an opaque lease/ownership token for a claimed message."""
|
|
20
|
+
return str(uuid.uuid4())
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Small state-machine guard shared by anything with start()/close() semantics.
|
|
2
|
+
|
|
3
|
+
Used by both ``SyncWorker`` (to reject a second concurrent ``start()``) and
|
|
4
|
+
``EdgeSync`` itself (to raise ``EdgeSyncNotStartedError`` when an operation
|
|
5
|
+
is attempted before ``start()`` has completed).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
from enum import Enum
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class LifecycleState(str, Enum):
|
|
15
|
+
IDLE = "idle"
|
|
16
|
+
STARTING = "starting"
|
|
17
|
+
RUNNING = "running"
|
|
18
|
+
STOPPING = "stopping"
|
|
19
|
+
STOPPED = "stopped"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LifecycleGuard:
|
|
23
|
+
"""Tracks start/stop transitions and makes them idempotent."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, name: str) -> None:
|
|
26
|
+
self._name = name
|
|
27
|
+
self._state = LifecycleState.IDLE
|
|
28
|
+
self._lock = asyncio.Lock()
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def state(self) -> LifecycleState:
|
|
32
|
+
return self._state
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def is_running(self) -> bool:
|
|
36
|
+
return self._state is LifecycleState.RUNNING
|
|
37
|
+
|
|
38
|
+
async def begin_start(self) -> bool:
|
|
39
|
+
"""Transition IDLE -> STARTING. Returns False if already started/starting."""
|
|
40
|
+
async with self._lock:
|
|
41
|
+
if self._state is not LifecycleState.IDLE:
|
|
42
|
+
return False
|
|
43
|
+
self._state = LifecycleState.STARTING
|
|
44
|
+
return True
|
|
45
|
+
|
|
46
|
+
def mark_running(self) -> None:
|
|
47
|
+
self._state = LifecycleState.RUNNING
|
|
48
|
+
|
|
49
|
+
async def begin_stop(self) -> bool:
|
|
50
|
+
"""Transition RUNNING -> STOPPING. Returns False if not running or already stopping."""
|
|
51
|
+
async with self._lock:
|
|
52
|
+
if self._state in (
|
|
53
|
+
LifecycleState.IDLE,
|
|
54
|
+
LifecycleState.STOPPING,
|
|
55
|
+
LifecycleState.STOPPED,
|
|
56
|
+
):
|
|
57
|
+
return False
|
|
58
|
+
self._state = LifecycleState.STOPPING
|
|
59
|
+
return True
|
|
60
|
+
|
|
61
|
+
def mark_stopped(self) -> None:
|
|
62
|
+
self._state = LifecycleState.STOPPED
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Controls the worker's polling cadence.
|
|
2
|
+
|
|
3
|
+
Polls again immediately when the previous batch found work (there may be
|
|
4
|
+
more queued up), and sleeps for ``poll_interval`` otherwise -- this is what
|
|
5
|
+
keeps an idle worker from busy-looping.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class PollScheduler:
|
|
14
|
+
def __init__(self, poll_interval: float) -> None:
|
|
15
|
+
self._poll_interval = poll_interval
|
|
16
|
+
|
|
17
|
+
async def wait(self, *, found_work: bool) -> None:
|
|
18
|
+
if found_work:
|
|
19
|
+
return
|
|
20
|
+
await asyncio.sleep(self._poll_interval)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""The background synchronization worker.
|
|
2
|
+
|
|
3
|
+
Each poll cycle:
|
|
4
|
+
|
|
5
|
+
1. Recover any messages whose lease expired (previous claim owner crashed
|
|
6
|
+
or was cancelled) and expire any messages past their TTL.
|
|
7
|
+
2. Claim a batch of eligible PENDING messages (storage-level ownership via
|
|
8
|
+
a lease fencing token -- see storage/sqlite.py).
|
|
9
|
+
3. Deliver each claimed message, bounded by ``worker_concurrency`` via a
|
|
10
|
+
semaphore so the number of in-flight network requests is capped.
|
|
11
|
+
4. Turn each ``DeliveryResult`` into a storage write: delivered, retry
|
|
12
|
+
scheduled, or dead-lettered.
|
|
13
|
+
|
|
14
|
+
The worker never treats in-memory state as the source of truth: every
|
|
15
|
+
decision it makes is persisted back to storage before the message is
|
|
16
|
+
considered handled, so a crash mid-batch just leaves messages IN_FLIGHT to
|
|
17
|
+
be recovered by the next lease sweep.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import asyncio
|
|
23
|
+
from datetime import timedelta
|
|
24
|
+
|
|
25
|
+
from edgesync.logging import logger
|
|
26
|
+
from edgesync.models.delivery import DeliveryOutcome, DeliveryResult
|
|
27
|
+
from edgesync.models.message import Message
|
|
28
|
+
from edgesync.retry.policy import RetryPolicy
|
|
29
|
+
from edgesync.storage.base import StorageBackend
|
|
30
|
+
from edgesync.transports.registry import TransportRegistry
|
|
31
|
+
from edgesync.utils.clock import Clock, SystemClock
|
|
32
|
+
from edgesync.worker.lifecycle import LifecycleGuard
|
|
33
|
+
from edgesync.worker.scheduler import PollScheduler
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class SyncWorker:
|
|
37
|
+
"""Polls storage for eligible messages and drives them to delivery."""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
storage: StorageBackend,
|
|
42
|
+
transports: TransportRegistry,
|
|
43
|
+
retry_policy: RetryPolicy,
|
|
44
|
+
*,
|
|
45
|
+
batch_size: int,
|
|
46
|
+
concurrency: int,
|
|
47
|
+
poll_interval: float,
|
|
48
|
+
lease_duration: float,
|
|
49
|
+
clock: Clock | None = None,
|
|
50
|
+
) -> None:
|
|
51
|
+
self._storage = storage
|
|
52
|
+
self._transports = transports
|
|
53
|
+
self._retry_policy = retry_policy
|
|
54
|
+
self._batch_size = batch_size
|
|
55
|
+
self._semaphore = asyncio.Semaphore(concurrency)
|
|
56
|
+
self._scheduler = PollScheduler(poll_interval)
|
|
57
|
+
self._lease_duration = lease_duration
|
|
58
|
+
self._clock = clock or SystemClock()
|
|
59
|
+
self._lifecycle = LifecycleGuard("SyncWorker")
|
|
60
|
+
self._task: asyncio.Task[None] | None = None
|
|
61
|
+
self._stopping = asyncio.Event()
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def is_running(self) -> bool:
|
|
65
|
+
return self._lifecycle.is_running
|
|
66
|
+
|
|
67
|
+
async def start(self) -> None:
|
|
68
|
+
if not await self._lifecycle.begin_start():
|
|
69
|
+
logger.debug("worker.start() called while already running; ignoring")
|
|
70
|
+
return
|
|
71
|
+
self._stopping.clear()
|
|
72
|
+
self._task = asyncio.create_task(self._run(), name="edgesync-worker")
|
|
73
|
+
self._lifecycle.mark_running()
|
|
74
|
+
logger.info("worker started")
|
|
75
|
+
|
|
76
|
+
async def stop(self, grace_period: float) -> None:
|
|
77
|
+
if not await self._lifecycle.begin_stop():
|
|
78
|
+
return
|
|
79
|
+
self._stopping.set()
|
|
80
|
+
if self._task is not None:
|
|
81
|
+
try:
|
|
82
|
+
await asyncio.wait_for(self._task, timeout=grace_period)
|
|
83
|
+
except asyncio.TimeoutError:
|
|
84
|
+
logger.warning(
|
|
85
|
+
"worker did not stop within %.1fs grace period; cancelling in-flight work",
|
|
86
|
+
grace_period,
|
|
87
|
+
)
|
|
88
|
+
except asyncio.CancelledError:
|
|
89
|
+
pass
|
|
90
|
+
self._task = None
|
|
91
|
+
self._lifecycle.mark_stopped()
|
|
92
|
+
logger.info("worker stopped")
|
|
93
|
+
|
|
94
|
+
async def _run(self) -> None:
|
|
95
|
+
while not self._stopping.is_set():
|
|
96
|
+
messages: list[Message] = []
|
|
97
|
+
try:
|
|
98
|
+
await self._storage.recover_expired_leases()
|
|
99
|
+
await self._storage.expire_stale_messages()
|
|
100
|
+
messages = await self._storage.claim_batch(self._batch_size, self._lease_duration)
|
|
101
|
+
except asyncio.CancelledError:
|
|
102
|
+
raise
|
|
103
|
+
except Exception:
|
|
104
|
+
logger.exception("worker: error while polling storage")
|
|
105
|
+
|
|
106
|
+
if messages:
|
|
107
|
+
logger.debug("worker: claimed %d message(s)", len(messages))
|
|
108
|
+
await asyncio.gather(*(self._process_one(m) for m in messages))
|
|
109
|
+
|
|
110
|
+
await self._scheduler.wait(found_work=bool(messages))
|
|
111
|
+
|
|
112
|
+
async def _process_one(self, message: Message) -> None:
|
|
113
|
+
async with self._semaphore:
|
|
114
|
+
try:
|
|
115
|
+
transport = self._transports.get(message.destination)
|
|
116
|
+
except Exception as exc:
|
|
117
|
+
logger.error("worker: no transport for message %s: %s", message.id, exc)
|
|
118
|
+
result = DeliveryResult.permanent_failure(f"no transport configured: {exc}")
|
|
119
|
+
else:
|
|
120
|
+
try:
|
|
121
|
+
result = await transport.deliver(message)
|
|
122
|
+
except asyncio.CancelledError:
|
|
123
|
+
raise
|
|
124
|
+
except Exception as exc:
|
|
125
|
+
logger.exception(
|
|
126
|
+
"worker: transport raised while delivering message %s", message.id
|
|
127
|
+
)
|
|
128
|
+
result = DeliveryResult.retryable_failure(f"transport raised: {exc}")
|
|
129
|
+
|
|
130
|
+
await self._handle_result(message, result)
|
|
131
|
+
|
|
132
|
+
async def _handle_result(self, message: Message, result: DeliveryResult) -> None:
|
|
133
|
+
assert message.lease_id is not None, "claimed messages always carry a lease_id"
|
|
134
|
+
|
|
135
|
+
if result.success:
|
|
136
|
+
await self._storage.mark_delivered(message.id, message.lease_id)
|
|
137
|
+
logger.info("message %s delivered (status=%s)", message.id, result.status_code)
|
|
138
|
+
return
|
|
139
|
+
|
|
140
|
+
new_attempts = message.attempts + 1
|
|
141
|
+
permanently_failed = (
|
|
142
|
+
result.outcome is DeliveryOutcome.PERMANENT_FAILURE
|
|
143
|
+
or self._retry_policy.is_exhausted(new_attempts)
|
|
144
|
+
)
|
|
145
|
+
if permanently_failed:
|
|
146
|
+
await self._storage.move_to_dead_letter(
|
|
147
|
+
message.id, message.lease_id, result.error or "delivery failed"
|
|
148
|
+
)
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
delay = self._retry_policy.next_delay(new_attempts)
|
|
152
|
+
next_attempt_at = self._clock.now() + timedelta(seconds=delay)
|
|
153
|
+
await self._storage.schedule_retry(
|
|
154
|
+
message.id, message.lease_id, result.error or "delivery failed", next_attempt_at
|
|
155
|
+
)
|
|
156
|
+
logger.info(
|
|
157
|
+
"message %s retry %d scheduled in %.2fs: %s",
|
|
158
|
+
message.id,
|
|
159
|
+
new_attempts,
|
|
160
|
+
delay,
|
|
161
|
+
result.error,
|
|
162
|
+
)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: edgesync
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Reliable data delivery for unreliable networks.
|
|
5
|
+
Project-URL: Homepage, https://github.com/adhuldas/EdgeSync
|
|
6
|
+
Project-URL: Repository, https://github.com/adhuldas/EdgeSync
|
|
7
|
+
Project-URL: Documentation, https://github.com/adhuldas/EdgeSync/tree/main/docs
|
|
8
|
+
Project-URL: Changelog, https://github.com/adhuldas/EdgeSync/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Adhul Das M K <adhulamz@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: edge,iot,offline,queue,reliability,retry,sync
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: aiosqlite>=0.20
|
|
26
|
+
Requires-Dist: httpx>=0.27
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# EdgeSync
|
|
30
|
+
|
|
31
|
+
[](https://github.com/adhuldas/EdgeSync/actions/workflows/ci.yml)
|
|
32
|
+
[](https://pypi.org/project/edgesync/)
|
|
33
|
+
[](LICENSE)
|
|
34
|
+
[](pyproject.toml)
|
|
35
|
+
|
|
36
|
+
A lightweight Python library for reliable data delivery from edge applications to cloud
|
|
37
|
+
services. EdgeSync provides persistent local queuing, store-and-forward synchronization,
|
|
38
|
+
automatic retries, and recovery from network or application failures. Built for IoT devices,
|
|
39
|
+
edge gateways, industrial systems, and any application operating with intermittent
|
|
40
|
+
connectivity.
|
|
41
|
+
|
|
42
|
+
## Why EdgeSync
|
|
43
|
+
|
|
44
|
+
Edge applications lose data for the same handful of reasons every time: the network drops,
|
|
45
|
+
the cloud endpoint is temporarily unavailable, or the process crashes mid-send. EdgeSync
|
|
46
|
+
solves this with a durable store-and-forward architecture:
|
|
47
|
+
|
|
48
|
+
1. Data is persisted locally (SQLite) before anything is sent.
|
|
49
|
+
2. A background worker attempts delivery to the configured destination.
|
|
50
|
+
3. Acknowledged messages are removed from the local queue.
|
|
51
|
+
4. Failed messages stay queued and are retried with backoff.
|
|
52
|
+
5. Pending data survives process crashes and device restarts.
|
|
53
|
+
|
|
54
|
+
EdgeSync provides **at-least-once** delivery, not exactly-once — see
|
|
55
|
+
[docs/reliability.md](docs/reliability.md) for the exact guarantee and how to build
|
|
56
|
+
idempotent consumers on top of it.
|
|
57
|
+
|
|
58
|
+
## Install
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install edgesync
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Quickstart
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import asyncio
|
|
68
|
+
from edgesync import EdgeSync
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
async def main():
|
|
72
|
+
async with EdgeSync(
|
|
73
|
+
database="edgesync.db",
|
|
74
|
+
endpoint="https://api.example.com/telemetry",
|
|
75
|
+
) as sync:
|
|
76
|
+
receipt = await sync.publish(
|
|
77
|
+
{
|
|
78
|
+
"device_id": "device-001",
|
|
79
|
+
"temperature": 28.5,
|
|
80
|
+
}
|
|
81
|
+
)
|
|
82
|
+
print(f"queued as {receipt.message_id}")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
asyncio.run(main())
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
That's it: `publish()` durably persists the message and returns as soon as it's on disk. A
|
|
89
|
+
background worker delivers it, retrying with exponential backoff on failure, without the
|
|
90
|
+
caller needing to stay connected or wait for the network.
|
|
91
|
+
|
|
92
|
+
## Usage with FastAPI, Flask, and plain asyncio
|
|
93
|
+
|
|
94
|
+
EdgeSync is async-native, so it's most at home in an `asyncio` app or an async framework like
|
|
95
|
+
FastAPI — start it once, publish from your handlers, close it on shutdown:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
# FastAPI
|
|
99
|
+
@asynccontextmanager
|
|
100
|
+
async def lifespan(app: FastAPI):
|
|
101
|
+
await sync.start()
|
|
102
|
+
yield
|
|
103
|
+
await sync.close()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
app = FastAPI(lifespan=lifespan)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@app.post("/telemetry")
|
|
110
|
+
async def telemetry(payload: dict):
|
|
111
|
+
receipt = await sync.publish(payload)
|
|
112
|
+
return {"message_id": receipt.message_id}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Flask is synchronous, so it needs a small bridge to run EdgeSync's event loop on a background
|
|
116
|
+
thread rather than spinning up a new loop per request. See
|
|
117
|
+
[docs/integrations.md](docs/integrations.md) for the full FastAPI, Flask, and plain-`asyncio`
|
|
118
|
+
guide, including the ready-to-use Flask bridge.
|
|
119
|
+
|
|
120
|
+
## Features
|
|
121
|
+
|
|
122
|
+
- **Durable local queue** — SQLite-backed, survives crashes and restarts.
|
|
123
|
+
- **At-least-once delivery** — messages are only removed once the destination acknowledges
|
|
124
|
+
them.
|
|
125
|
+
- **Automatic retries** — configurable exponential backoff with jitter.
|
|
126
|
+
- **Dead-letter queue** — messages that exhaust retries or fail permanently are retained for
|
|
127
|
+
inspection instead of being silently dropped.
|
|
128
|
+
- **Multiple destinations** — route different message types to different endpoints or
|
|
129
|
+
transports.
|
|
130
|
+
- **Pluggable transports** — ships with an HTTP transport; implement `Transport` for anything
|
|
131
|
+
else (MQTT, gRPC, a message broker, ...).
|
|
132
|
+
- **Bounded storage** — configurable queue capacity with a choice of overflow policies.
|
|
133
|
+
- **Async-native** — built on `asyncio` and `httpx`.
|
|
134
|
+
|
|
135
|
+
## Documentation
|
|
136
|
+
|
|
137
|
+
- [Getting started](docs/getting-started.md)
|
|
138
|
+
- [Using EdgeSync with FastAPI, Flask, and plain asyncio](docs/integrations.md)
|
|
139
|
+
- [Reliability & delivery guarantees](docs/reliability.md)
|
|
140
|
+
- [Storage design](docs/storage.md)
|
|
141
|
+
|
|
142
|
+
## Development
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
uv sync
|
|
146
|
+
uv run pytest
|
|
147
|
+
uv run ruff check .
|
|
148
|
+
uv run mypy
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
|
|
152
|
+
|
|
153
|
+
## License
|
|
154
|
+
|
|
155
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
edgesync/__init__.py,sha256=sEaXEvrOXlQrds-YbEYsoj83bIpu7EYuozpHtqQR0NM,1773
|
|
2
|
+
edgesync/client.py,sha256=3tQDfPrDYANaDcGXP3wJvAe215xCUDivY9vyAwP0La4,8905
|
|
3
|
+
edgesync/config.py,sha256=-KXf-Yigk_1rRjvQ8crJQVMZqRCtTnOesZg4cvgFhsw,2774
|
|
4
|
+
edgesync/exceptions.py,sha256=_v-ZnIYqHAz-Q76DVWmak0aZrqSwfqlpBioB6GYMGi8,1633
|
|
5
|
+
edgesync/logging.py,sha256=5qZE0zY6aNVVmjcY7Ge0sTj3CqzTktdkuWo7_ELxC-E,522
|
|
6
|
+
edgesync/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
edgesync/models/__init__.py,sha256=ha3zNRJCDE2lWvCcZfXoNvfsf4VO0lHz9PEr07TW398,471
|
|
8
|
+
edgesync/models/delivery.py,sha256=IQ2zxEnxdTI20m3Y8n2-YZEOKJGK1Mg3V7S8P66uH5U,1978
|
|
9
|
+
edgesync/models/message.py,sha256=j5XWth2oXrGfPGVWOQSEd1rwzQTSq_-LYWlDqe02kO4,2165
|
|
10
|
+
edgesync/models/receipt.py,sha256=aYr6yuJFo6hm3P8BY1w3gA0cU_VF2VTaxJSVm5K3DtI,606
|
|
11
|
+
edgesync/models/stats.py,sha256=BYbi-FdgC8RmggOutukOjgEQDhMynJsBQZUSEub70jo,737
|
|
12
|
+
edgesync/queue/__init__.py,sha256=FRTn9y5ffNnwqqeqXI8aZhq1sDwrJtfy7immIrumGHo,161
|
|
13
|
+
edgesync/queue/manager.py,sha256=35aF-Q3uKsSgEKg-BQk7rdho2yHyRZio-zX05L7U-UM,3102
|
|
14
|
+
edgesync/retry/__init__.py,sha256=QikUc49eKUD4cw9PiB9WN8SekilpESYRFIeFigdh49w,130
|
|
15
|
+
edgesync/retry/backoff.py,sha256=l_onIujEl7EdlaQ_Mg3KNhZ61XDUbI0XHpXwl5SghgU,1121
|
|
16
|
+
edgesync/retry/policy.py,sha256=Qny8xy_0w2yELtb533bpjZHo583VkQNuLMCh0eJNBpM,2101
|
|
17
|
+
edgesync/storage/__init__.py,sha256=haf9IUyapIrzYeQmhb7dLYt4S-LTfl7vyTpWn64ud2E,200
|
|
18
|
+
edgesync/storage/base.py,sha256=uL-wSC1CBiM8esQkoGfIZhnl0lloiskJgVg6MH0SVmA,3752
|
|
19
|
+
edgesync/storage/migrations.py,sha256=37WBCF0Q1ZpaObnsvGMLnTu1NSHoMP51RHkaRdxAqUQ,2211
|
|
20
|
+
edgesync/storage/sqlite.py,sha256=2ydkuZLaw2-P518EtYm-zk1Rdit9btnoAti7Os1vAxk,21281
|
|
21
|
+
edgesync/transports/__init__.py,sha256=sTW3K9XA1-DAzldfHCaGlv_XFcVnmMDtJHVnGJAl1gw,248
|
|
22
|
+
edgesync/transports/base.py,sha256=6kkZWHesNXzBafHrbrtVFn3k9NoDmCxIhjKWna-X2EQ,947
|
|
23
|
+
edgesync/transports/http.py,sha256=6q4ne70ztKAWTs0YJnqN72_3eo-NYFxEoD1qAwgoUek,4161
|
|
24
|
+
edgesync/transports/registry.py,sha256=5wl0EmkHRRTgZkct9xiLGPzdhkWj_SS5R9hzKvnqTf4,1695
|
|
25
|
+
edgesync/utils/__init__.py,sha256=iV5OXCznK1qG1K3zVCgjCV6H-X1ni2lhTI33dRNsDUw,67
|
|
26
|
+
edgesync/utils/clock.py,sha256=JbeVoS_3Tv85LiNrLaijdWIsNrsSrEmQxxZGcMf2ZF0,1363
|
|
27
|
+
edgesync/utils/ids.py,sha256=bES01uKJABuTR4XXgoctN-WfGRFCIDNzkyrqF2cINz4,507
|
|
28
|
+
edgesync/worker/__init__.py,sha256=KW3kb-fcSBsK-pMJeklhQ41H3GZEhtjr6lzMSE5PG2s,123
|
|
29
|
+
edgesync/worker/lifecycle.py,sha256=lp0h9bkt24TjVlYoIyEpvSR71sN1hbMuziydWKTF5JI,1851
|
|
30
|
+
edgesync/worker/scheduler.py,sha256=Zsr49wIVCl0laOJ-tsw57Cf0_e9SHK7chkiV3pMr7-c,556
|
|
31
|
+
edgesync/worker/sync_worker.py,sha256=3wL_Mo9MDJtLdq9_1yQRu3Q4WMGLE8S3ZLbjI3RPCN8,6367
|
|
32
|
+
edgesync-0.2.0.dist-info/METADATA,sha256=BIyrpIXVW5drKTJW8wiUYGsxOHor4Fhm-aGzTQsiafA,5545
|
|
33
|
+
edgesync-0.2.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
34
|
+
edgesync-0.2.0.dist-info/licenses/LICENSE,sha256=wfZl5iyvqgoar8g9FCkovkkWX_sPfEuD6NtW8nnUxUQ,1078
|
|
35
|
+
edgesync-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EdgeSync Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|