unique_toolkit 2026.48.0.dev12__py3-none-any.whl → 2026.48.0.dev14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unique_toolkit/experimental/resources/activity_records/README.md +284 -0
- unique_toolkit/experimental/resources/activity_records/__init__.py +8 -1
- unique_toolkit/experimental/resources/activity_records/config.py +8 -0
- unique_toolkit/experimental/resources/activity_records/enqueue.py +342 -0
- unique_toolkit/experimental/resources/activity_records/metrics.py +135 -0
- unique_toolkit/experimental/resources/activity_records/migrate.py +85 -0
- unique_toolkit/experimental/resources/activity_records/prepare.py +5 -0
- unique_toolkit/experimental/resources/activity_records/protocols.py +21 -0
- unique_toolkit/experimental/resources/activity_records/relay.py +526 -0
- unique_toolkit/experimental/resources/activity_records/store.py +354 -0
- unique_toolkit/monitoring/registry.py +18 -5
- {unique_toolkit-2026.48.0.dev12.dist-info → unique_toolkit-2026.48.0.dev14.dist-info}/METADATA +5 -2
- {unique_toolkit-2026.48.0.dev12.dist-info → unique_toolkit-2026.48.0.dev14.dist-info}/RECORD +14 -7
- {unique_toolkit-2026.48.0.dev12.dist-info → unique_toolkit-2026.48.0.dev14.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# Activity Records outbox
|
|
2
|
+
This library records validated Activity Records events in the producer's own
|
|
3
|
+
Postgres database and relays them through a sender the caller supplies. It
|
|
4
|
+
prepares events synchronously, stores them in a background task, and retries
|
|
5
|
+
failed deliveries. Rows are deleted only after the sender returns without raising,
|
|
6
|
+
or when the flag client's remote read says the company is off. A cached off defers
|
|
7
|
+
the rows until the cache expires and a remote read answers.
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
enqueue(event) -> ActivityEventOutbox table -> OutboxRelay -> sender
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
This is a port of the Node `@unique/activity-events-client` as on its master
|
|
14
|
+
branch: PR #33463 (merged) plus the follow-up #34009. It is experimental and
|
|
15
|
+
the interface may change.
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
```sh
|
|
19
|
+
uv add 'unique-toolkit[activity-records]'
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The extra installs asyncpg and prometheus_client. Also install `opentelemetry-api`
|
|
23
|
+
(or the `otel` extra) so events get their correlationId from the active trace.
|
|
24
|
+
|
|
25
|
+
## Migration
|
|
26
|
+
Run the idempotent migration before the application starts:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
python -m unique_toolkit.experimental.resources.activity_records.migrate
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
`DATABASE_URL` is required. The command exits non-zero, with a message that names the
|
|
33
|
+
table and no values, when the table exists but a column is missing or has a wrong
|
|
34
|
+
type. Extra columns are allowed. Run this command from a Helm
|
|
35
|
+
`pre-install,pre-upgrade` hook, not from application startup.
|
|
36
|
+
|
|
37
|
+
## Configuration
|
|
38
|
+
Required database variable:
|
|
39
|
+
|
|
40
|
+
- `DATABASE_URL`: asyncpg reads its query parameters. `sslmode=verify` becomes
|
|
41
|
+
`verify-full` and `connect_timeout` becomes the connect timeout. `application_name`
|
|
42
|
+
is kept. Any other parameter asyncpg does not read, such as Prisma's `schema=` or
|
|
43
|
+
`connection_limit=`, is dropped and only its name is logged once.
|
|
44
|
+
|
|
45
|
+
Size settings are `ACTIVITY_RECORDS_MAX_DATA_SIZE_BYTES` and
|
|
46
|
+
`ACTIVITY_RECORDS_MAX_BATCH_SIZE_BYTES`, defaulting to 1 MiB and 8 MiB.
|
|
47
|
+
|
|
48
|
+
The default feature flag client needs `CONFIGURATION_BACKEND_URL`,
|
|
49
|
+
`FEATURE_FLAG_SERVICE_ID`, and optionally `FEATURE_FLAG_CACHE_TTL_MS`. The flag
|
|
50
|
+
is `FEATURE_FLAG_ENABLE_ACTIVITY_RECORDS_EVENTS_UN_22616`. Keep
|
|
51
|
+
`FEATURE_FLAG_CACHE_TTL_MS` at 5000 or less, because the write path skips an insert
|
|
52
|
+
on a cached off. When no flag client is injected, the write path uses the shared
|
|
53
|
+
`FeatureFlagClient` and the relay builds its own instance from the same settings.
|
|
54
|
+
The relay needs its own cache: callers that keep the shared cache warm would hide
|
|
55
|
+
every remote read, and only a remote off deletes rows. An injected flag client is
|
|
56
|
+
used for both, so it must not share a cache that other callers keep warm.
|
|
57
|
+
|
|
58
|
+
## Sender
|
|
59
|
+
Delivery is done by a sender the caller supplies (`PublisherLike`):
|
|
60
|
+
`async publish(events: list[dict]) -> None`. Raise a `TimeoutError` subclass on
|
|
61
|
+
timeout. Any other exception means rejected and the row backs off. The caller
|
|
62
|
+
owns the sender's connection. The GraphQL sender lands with UN-29210.
|
|
63
|
+
|
|
64
|
+
## FastAPI
|
|
65
|
+
Create one runtime and start and stop it in the FastAPI lifespan:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from contextlib import asynccontextmanager
|
|
69
|
+
|
|
70
|
+
from fastapi import FastAPI
|
|
71
|
+
from unique_toolkit.experimental.resources.activity_records import ActivityRecordsOutbox
|
|
72
|
+
|
|
73
|
+
runtime = ActivityRecordsOutbox(publisher=my_sender)
|
|
74
|
+
|
|
75
|
+
@asynccontextmanager
|
|
76
|
+
async def lifespan(_app: FastAPI):
|
|
77
|
+
await runtime.start()
|
|
78
|
+
try:
|
|
79
|
+
yield
|
|
80
|
+
finally:
|
|
81
|
+
await runtime.stop()
|
|
82
|
+
|
|
83
|
+
app = FastAPI(lifespan=lifespan)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Use `runtime.is_ready()` for the service readiness probe. It answers false until
|
|
87
|
+
the relay's schema check passes, then stays true, also while the database is down
|
|
88
|
+
(the relay keeps polling and logs the errors). The check re-runs every 10 seconds,
|
|
89
|
+
like Node. An error during the check is logged once per outage as an
|
|
90
|
+
unreachable database, and only a real column mismatch is logged as a missing table.
|
|
91
|
+
|
|
92
|
+
## Enqueue an event
|
|
93
|
+
```python
|
|
94
|
+
from unique_toolkit.experimental.resources.activity_records import (
|
|
95
|
+
ActivityActor,
|
|
96
|
+
ActivityCloudEvent,
|
|
97
|
+
ActivityEventData,
|
|
98
|
+
enqueue,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
event = ActivityCloudEvent(
|
|
102
|
+
source="web-search",
|
|
103
|
+
type="web-search.tool.called",
|
|
104
|
+
data=ActivityEventData(
|
|
105
|
+
actor=ActivityActor(id="user-1", type="user"),
|
|
106
|
+
company_id="company-1",
|
|
107
|
+
),
|
|
108
|
+
)
|
|
109
|
+
enqueue(event)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The constructor rejects metadata over its length limit (identifier 128, ip 64,
|
|
113
|
+
text 256 UTF-16 units). `ActivityCloudEvent.build(source=..., type=..., data={...})`
|
|
114
|
+
shortens such values instead, so a long name never drops the record. It counts
|
|
115
|
+
the event as `truncated`, lists the fields in `event.truncated_fields` and logs a
|
|
116
|
+
warning with the event type, the source and the field paths (never the values).
|
|
117
|
+
The paths are wire paths like Node's, for example `actor.id`, `agent.id` and `targetName`.
|
|
118
|
+
Producers should build events with `build`.
|
|
119
|
+
|
|
120
|
+
`enqueue` is synchronous for the caller and never raises. Call it from the
|
|
121
|
+
event-loop thread. Preparing an event (validation, sizing, truncation) is CPU bound
|
|
122
|
+
and runs synchronously on the event loop, like Node's. It is best effort:
|
|
123
|
+
the prepared insert runs in a background task, and normal shutdown drains
|
|
124
|
+
pending inserts for up to 10 seconds. Delivery is at least once. The insert
|
|
125
|
+
does not wait for the relay's table check; a missing or mismatched table fails
|
|
126
|
+
the insert and counts a record failure. It waits at most 5 seconds for the
|
|
127
|
+
database pool to exist. Waiting for a free connection is bounded by the 40 second
|
|
128
|
+
insert timeout. `enqueue` skips the insert when a fresh or cached flag read says
|
|
129
|
+
off. After a failed flag read (including the 5 second timeout) a company is not
|
|
130
|
+
asked again for 15 seconds and its events are inserted; the relay checks the
|
|
131
|
+
flag again before delivery and defers when it is unknown. A stale or fallback answer
|
|
132
|
+
starts the same pause. At most 1000 inserts wait at once; further events are dropped
|
|
133
|
+
and counted in `record_failures`.
|
|
134
|
+
|
|
135
|
+
Events are lost, and counted in `record_failures`, when the insert fails. A refused
|
|
136
|
+
connection loses the event at once. The 5 seconds cover only the wait for the
|
|
137
|
+
database pool. An insert on a frozen database fails after at most 40 seconds (10
|
|
138
|
+
seconds to connect and 30 seconds per command; the insert call is limited to 40
|
|
139
|
+
seconds). Events are also lost when the process exits before the background
|
|
140
|
+
insert runs (not counted). An `enqueue` call while no outbox is running (before
|
|
141
|
+
`start()`, or while `stop()` runs) logs "no Activity Records outbox is running" and
|
|
142
|
+
counts in `record_failures` once an outbox has run in the process.
|
|
143
|
+
|
|
144
|
+
When the flag is off at `enqueue`, the insert is skipped without a metric or a log,
|
|
145
|
+
like Node's best effort write. The 15 second pause on flag reads starts when the
|
|
146
|
+
5 second wait expires, the flag client raises, or it answers stale or fallback. The
|
|
147
|
+
default flag client never raises: when a request fails or exceeds its own 2 second
|
|
148
|
+
timeout (1 second to connect) it returns a fallback or stale answer, which
|
|
149
|
+
the write path treats as unknown and inserts the event.
|
|
150
|
+
|
|
151
|
+
## criticalData size limit
|
|
152
|
+
An event's `data` may not exceed `ACTIVITY_RECORDS_MAX_DATA_SIZE_BYTES` (1 MiB by
|
|
153
|
+
default). When it does, `criticalData` is cut to fit: strings are shortened, keys
|
|
154
|
+
and array items can be dropped, and `criticalData.truncated` is set to true. The
|
|
155
|
+
event is counted as `truncated`. A value nested deeper than 200 levels is cut the same
|
|
156
|
+
way: the part below that depth becomes null and `criticalData.truncated` marks it too.
|
|
157
|
+
|
|
158
|
+
## Kill switch
|
|
159
|
+
When the relay reads a company's flag as off from the remote, it deletes that
|
|
160
|
+
company's queued rows without publishing them and counts them in
|
|
161
|
+
`unique_activity_event_outbox_discarded_total`. This also hits events recorded while
|
|
162
|
+
the flag was on. The configuration backend answers false for a flag key it does not
|
|
163
|
+
know, so a flag key that does not exist reads as off and deletes every queued event
|
|
164
|
+
of every company. Roll out in this order: release the code that reads the flag,
|
|
165
|
+
create the flag, and only then enable it. To retire the flag, remove the flag check
|
|
166
|
+
from the service first and the key from the registry afterwards.
|
|
167
|
+
|
|
168
|
+
## Personal data in the outbox
|
|
169
|
+
The `payload` column holds the full event, including `criticalData` (for web search
|
|
170
|
+
the user prompt and the tool arguments). It is a second store of personal data and
|
|
171
|
+
is not covered by the data redaction job. A healthy outbox holds seconds of it. A
|
|
172
|
+
stuck row stays until it is delivered. Company deletion does not purge outbox rows.
|
|
173
|
+
|
|
174
|
+
## Rolling back
|
|
175
|
+
A rollback never removes the table. Queued rows stay, and nothing delivers them
|
|
176
|
+
while the service runs without this library. Wait for
|
|
177
|
+
`unique_activity_event_outbox_pending_rows` to reach 0 before rolling back.
|
|
178
|
+
|
|
179
|
+
## Settings
|
|
180
|
+
`ACTIVITY_RECORDS_MAX_DATA_SIZE_BYTES` and `ACTIVITY_RECORDS_MAX_BATCH_SIZE_BYTES`
|
|
181
|
+
are read by the activity-records intake too. If you override either, override it on
|
|
182
|
+
both sides. A missing value uses the default and a numeric string is accepted. A
|
|
183
|
+
value that is not a number, or a batch size too small for the data size, makes
|
|
184
|
+
`start()` raise before anything is created, like Node's assert at construction.
|
|
185
|
+
|
|
186
|
+
The relay's timings are fixed, like Node's. It claims up to 100 rows under a 90
|
|
187
|
+
second lease and publishes nothing after half of it. It polls every second when idle
|
|
188
|
+
and waits 5 seconds for a flag answer and 30 seconds for the sender. Rows whose flag
|
|
189
|
+
cannot be read are deferred by 15 to 30 seconds (30 seconds with 50 to 100 percent
|
|
190
|
+
jitter), and that company is not asked again for 15 seconds. Gauges refresh every 15
|
|
191
|
+
seconds and keep their last value if a refresh fails. The relay claims again at once
|
|
192
|
+
only after settling a full batch; a round that only deferred rows waits the idle
|
|
193
|
+
poll. On shutdown the relay stops and hands its rows back, then pending inserts get
|
|
194
|
+
10 seconds, then the pool closes (terminated after 5 seconds if the database does
|
|
195
|
+
not answer).
|
|
196
|
+
|
|
197
|
+
## Alerts
|
|
198
|
+
The Helm library chart `charts/activity-events-client` ships the alerts for the
|
|
199
|
+
outbox metrics. A service that uses this library includes that chart and sets
|
|
200
|
+
`activityEventsClient.metricName` to the `metric_name` used in code (default
|
|
201
|
+
`unique_activity_events_total`). Keep `tableAlerts` true on exactly one deployable
|
|
202
|
+
per outbox table. The backoff cap alert is critical. Do not write alerts on these
|
|
203
|
+
metrics by hand in `prometheus.additionalAlerts`, because the chart already renders
|
|
204
|
+
them and a copy fires twice. Add the chart directory to the service's CI path
|
|
205
|
+
filters and sparse checkout. The alert texts mention RabbitMQ today; read it as the
|
|
206
|
+
sender. A `timeout` publish failure means the sender raised a `TimeoutError` or took
|
|
207
|
+
longer than 30 seconds, and `rejected` means any other exception.
|
|
208
|
+
The alert texts need new wording when the GraphQL sender replaces RabbitMQ.
|
|
209
|
+
|
|
210
|
+
## Readiness and migration
|
|
211
|
+
Run the migration as a Helm pre-install and pre-upgrade hook so there is no deploy
|
|
212
|
+
order problem. Extra columns are allowed, so a rollback onto a newer migration keeps
|
|
213
|
+
working. Until the schema check passes the relay does not run and `is_ready()` is
|
|
214
|
+
false.
|
|
215
|
+
|
|
216
|
+
## Metrics
|
|
217
|
+
- `unique_activity_events_total{outcome,source,actorType,eventType}`
|
|
218
|
+
- `unique_activity_event_outbox_record_failures_total`
|
|
219
|
+
- `unique_activity_event_outbox_publish_failures_total{reason}`
|
|
220
|
+
- `unique_activity_event_outbox_backoff_max_reached_total`
|
|
221
|
+
- `unique_activity_event_outbox_discarded_total`
|
|
222
|
+
- `unique_activity_event_outbox_invalid_events_total`
|
|
223
|
+
- `unique_activity_event_outbox_flag_check_failures_total`
|
|
224
|
+
- `unique_activity_event_outbox_pending_rows`
|
|
225
|
+
- `unique_activity_event_outbox_oldest_pending_age_seconds`
|
|
226
|
+
- `unique_activity_event_outbox_rows_at_max_backoff`
|
|
227
|
+
|
|
228
|
+
The metrics use the toolkit's private registry. The host must serve
|
|
229
|
+
`unique_toolkit.monitoring.get_metrics()` on the `/metrics` path that Prometheus
|
|
230
|
+
scrapes. The three gauges (`pending_rows`, `oldest_pending_age_seconds`, `rows_at_max_backoff`)
|
|
231
|
+
are table-wide, so every worker reports the same value; in multiprocess mode they use
|
|
232
|
+
`livemax` to keep one series. Publish failure reasons are `timeout` and `rejected`. A publish that fails while
|
|
233
|
+
the relay is stopping is neither counted nor backed off, because `stop()` hands the
|
|
234
|
+
rows back. A sender call that takes longer than 30 seconds counts as a timeout. A
|
|
235
|
+
rejected batch is retried row by row, like Node, so a rejecting sender gets one
|
|
236
|
+
call per row.
|
|
237
|
+
|
|
238
|
+
## Operations
|
|
239
|
+
The outbox table belongs to the producer's database. There is no TTL or cleanup
|
|
240
|
+
job. A row the sender keeps rejecting retries forever, with a 15 minute cap,
|
|
241
|
+
and needs manual deletion. Every retry delay is jittered to 50 to 100 percent, so a
|
|
242
|
+
retry at the cap comes after 7.5 to 15 minutes.
|
|
243
|
+
|
|
244
|
+
## Differences from the Node library
|
|
245
|
+
- A stale "on" flag is treated as on.
|
|
246
|
+
- The flag client has no cache bypass argument. Node deletes after an uncached read; here
|
|
247
|
+
a cached off only defers, and a delete needs an off from the remote read.
|
|
248
|
+
- Node answers an unknown flag with a rejection. The Python flag client returns a fallback
|
|
249
|
+
or stale answer instead, and the relay treats both as unknown. The write path starts its
|
|
250
|
+
15 second pause on such an answer too, as on an error or the 5 second timeout.
|
|
251
|
+
- The write path has no 5 second cache of its own; it reads through the flag client's
|
|
252
|
+
cache on every insert, except for 15 seconds after a failed, stale or fallback read,
|
|
253
|
+
when it skips the read and inserts.
|
|
254
|
+
- The pool has a 30 second command timeout and a 10 second connect timeout, and an insert
|
|
255
|
+
task is limited to 40 seconds. Node has none of these.
|
|
256
|
+
- `stop()` terminates a pool that does not close within 5 seconds, and cancels the running
|
|
257
|
+
round after 5 seconds where Node only stops waiting for it.
|
|
258
|
+
- `start()` returns before the first schema check, so `is_ready()` is false right after
|
|
259
|
+
it; Node awaits the first check.
|
|
260
|
+
- A failed schema check is logged once per outage as an unreachable database and never as
|
|
261
|
+
a mismatch; Node warns on every failed check.
|
|
262
|
+
- The write path awaits a flag read of up to 5 seconds in the background insert task
|
|
263
|
+
before inserting; Node decides synchronously from its own cache, records on unknown and
|
|
264
|
+
refreshes in the background.
|
|
265
|
+
- `sslmode=verify` in `DATABASE_URL` is translated to asyncpg's `verify-full`.
|
|
266
|
+
- The migration connection has a 60 second command timeout, so a held advisory lock or a
|
|
267
|
+
stuck DDL statement fails the Helm hook instead of waiting forever. Node has none.
|
|
268
|
+
- `prepare_rows` rejects a company id that sanitization would change and an event of
|
|
269
|
+
another company; Node sanitizes and does not compare.
|
|
270
|
+
- The relay times out a sender call after 30 seconds; Node's publisher has its own 30
|
|
271
|
+
second confirm timeout.
|
|
272
|
+
- The relay builds its own flag client when none is injected (see Configuration); Node's
|
|
273
|
+
relay and write path share the host's `isEnabled`.
|
|
274
|
+
- There is no `record_in_transaction`.
|
|
275
|
+
- The relay's schema check uses the same strict keys as zod, but parses the event id and
|
|
276
|
+
time more leniently: an id without hyphens or a time with a numeric offset such as
|
|
277
|
+
`+00:00` still passes.
|
|
278
|
+
- The JSON nesting depth limit is 200 (Node: 500).
|
|
279
|
+
- Numbers are sized as Postgres jsonb prints them, because Python reads payloads back with
|
|
280
|
+
`json.loads`: a float with an exponent, such as `1e300`, comes back as a long integer.
|
|
281
|
+
Node does not need this.
|
|
282
|
+
- `ActivityCloudEvent.truncated_fields` and the truncation warning use wire paths like
|
|
283
|
+
Node (`actor.id`, `targetName`).
|
|
284
|
+
- No broker publisher in this package yet.
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Public API for the experimental Activity Records outbox."""
|
|
2
2
|
|
|
3
|
+
from .enqueue import ActivityRecordsOutbox, drain, enqueue, is_running
|
|
4
|
+
from .relay import OutboxRelay
|
|
3
5
|
from .schemas import (
|
|
4
6
|
ACTIVITY_METADATA_MAX_LENGTH,
|
|
5
7
|
ActivityActor,
|
|
@@ -14,4 +16,9 @@ __all__ = [
|
|
|
14
16
|
"ActivityAgent",
|
|
15
17
|
"ActivityCloudEvent",
|
|
16
18
|
"ActivityEventData",
|
|
19
|
+
"ActivityRecordsOutbox",
|
|
20
|
+
"OutboxRelay",
|
|
21
|
+
"drain",
|
|
22
|
+
"enqueue",
|
|
23
|
+
"is_running",
|
|
17
24
|
]
|
|
@@ -16,6 +16,14 @@ from unique_toolkit.experimental.resources.activity_records.schemas import (
|
|
|
16
16
|
|
|
17
17
|
DEFAULT_MAX_DATA_SIZE_BYTES: Final = 1_048_576
|
|
18
18
|
DEFAULT_MAX_BATCH_SIZE_BYTES: Final = 8_388_608
|
|
19
|
+
FEATURE_FLAG_ENABLE_ACTIVITY_RECORDS_EVENTS_UN_22616: Final = (
|
|
20
|
+
"FEATURE_FLAG_ENABLE_ACTIVITY_RECORDS_EVENTS_UN_22616"
|
|
21
|
+
)
|
|
22
|
+
OUTBOX_MAX_BACKOFF_ATTEMPTS: Final = 11
|
|
23
|
+
DEFAULT_ACTIVITY_EVENTS_METRIC_NAME: Final = "unique_activity_events_total"
|
|
24
|
+
ACTIVITY_RECORDS_INSTALL_HINT: Final = (
|
|
25
|
+
"Install it with: uv add 'unique-toolkit[activity-records]'"
|
|
26
|
+
)
|
|
19
27
|
|
|
20
28
|
_MAX_SAFE_INTEGER: Final = 2**53 - 1
|
|
21
29
|
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Best-effort Activity Records enqueueing and process lifecycle."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import contextlib
|
|
7
|
+
import functools
|
|
8
|
+
import logging
|
|
9
|
+
import os
|
|
10
|
+
import time
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import TYPE_CHECKING, cast
|
|
13
|
+
|
|
14
|
+
from unique_toolkit.experimental.resources.activity_records import (
|
|
15
|
+
prepare as prepare_module,
|
|
16
|
+
)
|
|
17
|
+
from unique_toolkit.experimental.resources.activity_records.config import (
|
|
18
|
+
DEFAULT_ACTIVITY_EVENTS_METRIC_NAME,
|
|
19
|
+
FEATURE_FLAG_ENABLE_ACTIVITY_RECORDS_EVENTS_UN_22616,
|
|
20
|
+
ActivityRecordsConfig,
|
|
21
|
+
)
|
|
22
|
+
from unique_toolkit.experimental.resources.activity_records.prepare import (
|
|
23
|
+
OutboxRow,
|
|
24
|
+
WireEvent,
|
|
25
|
+
prepare_rows,
|
|
26
|
+
to_last_error,
|
|
27
|
+
)
|
|
28
|
+
from unique_toolkit.experimental.resources.activity_records.protocols import (
|
|
29
|
+
FlagClientLike,
|
|
30
|
+
PublisherLike,
|
|
31
|
+
)
|
|
32
|
+
from unique_toolkit.experimental.resources.activity_records.relay import (
|
|
33
|
+
OutboxRelay,
|
|
34
|
+
count_events,
|
|
35
|
+
)
|
|
36
|
+
from unique_toolkit.experimental.resources.activity_records.schemas import (
|
|
37
|
+
ActivityCloudEvent,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
if TYPE_CHECKING:
|
|
41
|
+
import asyncpg # pyright: ignore[reportMissingImports]
|
|
42
|
+
|
|
43
|
+
from unique_toolkit.experimental.resources.activity_records.metrics import (
|
|
44
|
+
ActivityEventOutboxMetrics,
|
|
45
|
+
)
|
|
46
|
+
from unique_toolkit.experimental.resources.activity_records.store import OutboxStore
|
|
47
|
+
|
|
48
|
+
logger = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
_DEFAULT_DRAIN_TIMEOUT_SECONDS = 10.0
|
|
51
|
+
_FIRST_FLAG_READ_TIMEOUT_SECONDS = 5.0
|
|
52
|
+
POOL_WAIT_TIMEOUT_SECONDS = 5
|
|
53
|
+
POOL_CLOSE_TIMEOUT_SECONDS = 5
|
|
54
|
+
# The pool's 10 s connect and 30 s command timeouts, so a waiting insert is bounded too.
|
|
55
|
+
INSERT_TIMEOUT_SECONDS = 40
|
|
56
|
+
WRITE_FLAG_FAILURE_COOLDOWN_SECONDS = 15
|
|
57
|
+
MAX_PENDING_INSERTS = 1000
|
|
58
|
+
_pending_inserts: set[asyncio.Task[None]] = set()
|
|
59
|
+
_active_context: _EnqueueContext | None = None
|
|
60
|
+
# Kept after stop() so late enqueue calls still count in record_failures.
|
|
61
|
+
_last_metrics: ActivityEventOutboxMetrics | None = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(eq=False)
|
|
65
|
+
class _EnqueueContext:
|
|
66
|
+
"""What `enqueue` needs; `store_future` resolves once the pool exists."""
|
|
67
|
+
|
|
68
|
+
metrics: ActivityEventOutboxMetrics
|
|
69
|
+
flag_client: FlagClientLike
|
|
70
|
+
max_data_size_bytes: int
|
|
71
|
+
store_future: asyncio.Future[OutboxStore]
|
|
72
|
+
flag_failed_until: dict[str, float] = field(default_factory=dict)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _PendingInsertLimit(Exception):
|
|
76
|
+
"""Too many inserts are waiting, so the event is dropped."""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _record_failure(
|
|
80
|
+
metrics: ActivityEventOutboxMetrics | None,
|
|
81
|
+
company_id: str,
|
|
82
|
+
event_count: int,
|
|
83
|
+
error: BaseException,
|
|
84
|
+
) -> None:
|
|
85
|
+
if metrics is not None:
|
|
86
|
+
metrics.record_failures.inc(event_count)
|
|
87
|
+
logger.error(
|
|
88
|
+
"Could not record activity events in the outbox; they are lost "
|
|
89
|
+
"(companyId=%s eventCount=%s errorName=%s)",
|
|
90
|
+
company_id,
|
|
91
|
+
event_count,
|
|
92
|
+
to_last_error(error),
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def _close_pool(pool: asyncpg.Pool) -> None:
|
|
97
|
+
"""Close the pool, terminating it when a frozen database keeps ``close`` waiting."""
|
|
98
|
+
try:
|
|
99
|
+
await asyncio.wait_for(pool.close(), POOL_CLOSE_TIMEOUT_SECONDS)
|
|
100
|
+
except TimeoutError:
|
|
101
|
+
pool.terminate()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
async def _flag_is_off(write: _EnqueueContext, company_id: str) -> bool:
|
|
105
|
+
"""True only for a fresh or cached off; failed, stale or fallback reads start a cooldown."""
|
|
106
|
+
failed_until = write.flag_failed_until.get(company_id)
|
|
107
|
+
if failed_until is not None:
|
|
108
|
+
if time.monotonic() < failed_until:
|
|
109
|
+
return False
|
|
110
|
+
del write.flag_failed_until[company_id]
|
|
111
|
+
try:
|
|
112
|
+
evaluation = await asyncio.wait_for(
|
|
113
|
+
write.flag_client.evaluate(
|
|
114
|
+
FEATURE_FLAG_ENABLE_ACTIVITY_RECORDS_EVENTS_UN_22616,
|
|
115
|
+
company_id=company_id,
|
|
116
|
+
),
|
|
117
|
+
_FIRST_FLAG_READ_TIMEOUT_SECONDS,
|
|
118
|
+
)
|
|
119
|
+
except Exception as error:
|
|
120
|
+
write.flag_failed_until[company_id] = (
|
|
121
|
+
time.monotonic() + WRITE_FLAG_FAILURE_COOLDOWN_SECONDS
|
|
122
|
+
)
|
|
123
|
+
logger.warning(
|
|
124
|
+
"Could not refresh the activity records flag (companyId=%s errorName=%s)",
|
|
125
|
+
company_id,
|
|
126
|
+
to_last_error(error),
|
|
127
|
+
)
|
|
128
|
+
return False
|
|
129
|
+
if evaluation.reason in ("stale", "fallback"):
|
|
130
|
+
write.flag_failed_until[company_id] = (
|
|
131
|
+
time.monotonic() + WRITE_FLAG_FAILURE_COOLDOWN_SECONDS
|
|
132
|
+
)
|
|
133
|
+
return evaluation.value is False and evaluation.reason in ("remote", "cached")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
async def _insert(
|
|
137
|
+
write: _EnqueueContext,
|
|
138
|
+
rows: list[OutboxRow],
|
|
139
|
+
truncated_events: list[WireEvent],
|
|
140
|
+
company_id: str,
|
|
141
|
+
) -> None:
|
|
142
|
+
try:
|
|
143
|
+
store = await asyncio.wait_for(
|
|
144
|
+
asyncio.shield(write.store_future), POOL_WAIT_TIMEOUT_SECONDS
|
|
145
|
+
)
|
|
146
|
+
if await _flag_is_off(write, company_id):
|
|
147
|
+
return
|
|
148
|
+
await asyncio.wait_for(store.insert(rows), INSERT_TIMEOUT_SECONDS)
|
|
149
|
+
if truncated_events:
|
|
150
|
+
count_events(write.metrics, "truncated", truncated_events)
|
|
151
|
+
except Exception as error:
|
|
152
|
+
_record_failure(write.metrics, company_id, len(rows), error)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def is_running() -> bool:
|
|
156
|
+
"""Whether an Activity Records outbox is accepting events."""
|
|
157
|
+
return _active_context is not None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def enqueue(event: ActivityCloudEvent) -> None:
|
|
161
|
+
"""Prepare and enqueue one event from the event-loop thread without awaiting it."""
|
|
162
|
+
company_id = event.data.company_id
|
|
163
|
+
write = _active_context
|
|
164
|
+
if write is None:
|
|
165
|
+
if _last_metrics is not None:
|
|
166
|
+
_last_metrics.record_failures.inc()
|
|
167
|
+
logger.error(
|
|
168
|
+
"Could not record an activity event: no Activity Records outbox is running "
|
|
169
|
+
"(it is not started, or it is stopping); the event is lost (companyId=%s)",
|
|
170
|
+
company_id,
|
|
171
|
+
)
|
|
172
|
+
return
|
|
173
|
+
try:
|
|
174
|
+
asyncio.get_running_loop()
|
|
175
|
+
if len(_pending_inserts) >= MAX_PENDING_INSERTS:
|
|
176
|
+
raise _PendingInsertLimit
|
|
177
|
+
if event.truncated_fields:
|
|
178
|
+
logger.warning(
|
|
179
|
+
"Activity event metadata field(s) truncated to fit their length limit: "
|
|
180
|
+
"eventType=%s source=%s metadataFields=%s",
|
|
181
|
+
event.type,
|
|
182
|
+
event.source,
|
|
183
|
+
event.truncated_fields,
|
|
184
|
+
)
|
|
185
|
+
prepared = prepare_rows(company_id, [event], write.max_data_size_bytes)
|
|
186
|
+
truncated_events = prepared.truncated_events
|
|
187
|
+
if event.truncated_fields and not truncated_events:
|
|
188
|
+
truncated_events = [prepared.rows[0].payload]
|
|
189
|
+
task = asyncio.create_task(
|
|
190
|
+
_insert(write, prepared.rows, truncated_events, company_id)
|
|
191
|
+
)
|
|
192
|
+
_pending_inserts.add(task)
|
|
193
|
+
task.add_done_callback(_pending_inserts.discard)
|
|
194
|
+
except Exception as error:
|
|
195
|
+
_record_failure(write.metrics, company_id, 1, error)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
async def drain(timeout: float = _DEFAULT_DRAIN_TIMEOUT_SECONDS) -> None:
|
|
199
|
+
"""Wait for currently scheduled inserts, leaving timed-out tasks running."""
|
|
200
|
+
if not _pending_inserts:
|
|
201
|
+
return
|
|
202
|
+
pending = asyncio.gather(*_pending_inserts, return_exceptions=True)
|
|
203
|
+
with contextlib.suppress(TimeoutError):
|
|
204
|
+
await asyncio.wait_for(asyncio.shield(pending), timeout)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@functools.cache
|
|
208
|
+
def _relay_flag_client() -> FlagClientLike:
|
|
209
|
+
"""Own cache, or producers keep it warm and the relay never sees a remote read."""
|
|
210
|
+
from unique_toolkit.experimental.resources.feature_flags.client import (
|
|
211
|
+
FeatureFlagClient,
|
|
212
|
+
)
|
|
213
|
+
from unique_toolkit.experimental.resources.feature_flags.settings import (
|
|
214
|
+
FeatureFlagSettings,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
settings = FeatureFlagSettings()
|
|
218
|
+
# start() already validated these settings through from_settings().
|
|
219
|
+
return FeatureFlagClient(
|
|
220
|
+
url=cast("str", settings.configuration_backend_url),
|
|
221
|
+
service_id=cast("str", settings.feature_flag_service_id),
|
|
222
|
+
ttl_ms=settings.feature_flag_cache_ttl_ms,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class ActivityRecordsOutbox:
|
|
227
|
+
"""Own the database pool, relay, and enqueue lifecycle; the caller supplies the sender."""
|
|
228
|
+
|
|
229
|
+
def __init__(
|
|
230
|
+
self,
|
|
231
|
+
*,
|
|
232
|
+
publisher: PublisherLike,
|
|
233
|
+
flag_client: FlagClientLike | None = None,
|
|
234
|
+
metrics: ActivityEventOutboxMetrics | None = None,
|
|
235
|
+
metric_name: str = DEFAULT_ACTIVITY_EVENTS_METRIC_NAME,
|
|
236
|
+
) -> None:
|
|
237
|
+
self.publisher = publisher
|
|
238
|
+
self.flag_client = flag_client
|
|
239
|
+
self.metrics = metrics
|
|
240
|
+
self.metric_name = metric_name
|
|
241
|
+
self.pool: asyncpg.Pool | None = None
|
|
242
|
+
self.relay: OutboxRelay | None = None
|
|
243
|
+
self._write: _EnqueueContext | None = None
|
|
244
|
+
self._stop_lock = asyncio.Lock()
|
|
245
|
+
|
|
246
|
+
async def start(self) -> None:
|
|
247
|
+
"""Create dependencies from environment and start the relay."""
|
|
248
|
+
global _active_context, _last_metrics
|
|
249
|
+
if self._write is not None:
|
|
250
|
+
if _active_context is self._write:
|
|
251
|
+
return
|
|
252
|
+
raise RuntimeError(
|
|
253
|
+
"The Activity Records outbox is stopping; "
|
|
254
|
+
"await stop() before calling start() again"
|
|
255
|
+
)
|
|
256
|
+
if _active_context is not None:
|
|
257
|
+
raise RuntimeError("Only one Activity Records outbox can be active")
|
|
258
|
+
if prepare_module.otel_trace is None:
|
|
259
|
+
logger.warning(
|
|
260
|
+
"opentelemetry-api is not installed, so events get no correlationId from "
|
|
261
|
+
"the active trace. Install it with: uv add opentelemetry-api"
|
|
262
|
+
)
|
|
263
|
+
settings = ActivityRecordsConfig()
|
|
264
|
+
database_url = os.environ.get("DATABASE_URL")
|
|
265
|
+
if not database_url:
|
|
266
|
+
raise ValueError("DATABASE_URL is required for the Activity Records outbox")
|
|
267
|
+
from unique_toolkit.experimental.resources.activity_records.metrics import (
|
|
268
|
+
get_or_create_activity_event_outbox_metrics,
|
|
269
|
+
)
|
|
270
|
+
from unique_toolkit.experimental.resources.activity_records.store import (
|
|
271
|
+
OutboxStore,
|
|
272
|
+
create_pool,
|
|
273
|
+
)
|
|
274
|
+
from unique_toolkit.experimental.resources.feature_flags.client import (
|
|
275
|
+
FeatureFlagClient,
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
flag_client = self.flag_client or FeatureFlagClient.from_settings()
|
|
279
|
+
relay_flag_client = self.flag_client or _relay_flag_client()
|
|
280
|
+
metrics = self.metrics or get_or_create_activity_event_outbox_metrics(
|
|
281
|
+
metric_name=self.metric_name
|
|
282
|
+
)
|
|
283
|
+
write = _EnqueueContext(
|
|
284
|
+
metrics,
|
|
285
|
+
flag_client,
|
|
286
|
+
settings.max_data_size_bytes,
|
|
287
|
+
asyncio.get_running_loop().create_future(),
|
|
288
|
+
)
|
|
289
|
+
pool = None
|
|
290
|
+
_active_context = self._write = write
|
|
291
|
+
_last_metrics = metrics
|
|
292
|
+
try:
|
|
293
|
+
pool = await create_pool(database_url, min_size=0, max_size=4)
|
|
294
|
+
outbox_store = OutboxStore(pool)
|
|
295
|
+
write.store_future.set_result(outbox_store)
|
|
296
|
+
relay = OutboxRelay(
|
|
297
|
+
outbox_store,
|
|
298
|
+
self.publisher,
|
|
299
|
+
relay_flag_client,
|
|
300
|
+
metrics,
|
|
301
|
+
max_batch_size_bytes=settings.max_batch_size_bytes,
|
|
302
|
+
max_data_size_bytes=settings.max_data_size_bytes,
|
|
303
|
+
)
|
|
304
|
+
await relay.start()
|
|
305
|
+
except BaseException:
|
|
306
|
+
if _active_context is write:
|
|
307
|
+
_active_context = None
|
|
308
|
+
if self._write is write:
|
|
309
|
+
self._write = None
|
|
310
|
+
if pool is not None:
|
|
311
|
+
await _close_pool(pool)
|
|
312
|
+
raise
|
|
313
|
+
if _active_context is not write: # stop() ran while we were starting
|
|
314
|
+
await relay.stop()
|
|
315
|
+
await _close_pool(pool)
|
|
316
|
+
return
|
|
317
|
+
self.pool, self.relay = pool, relay
|
|
318
|
+
|
|
319
|
+
def is_ready(self) -> bool:
|
|
320
|
+
return self.relay is not None and self.relay.is_ready()
|
|
321
|
+
|
|
322
|
+
async def stop(self) -> None:
|
|
323
|
+
"""Stop delivery, drain inserts, then close the database pool."""
|
|
324
|
+
global _active_context
|
|
325
|
+
write = self._write
|
|
326
|
+
async with self._stop_lock:
|
|
327
|
+
if self._write is not write: # already stopped, or replaced by a restart
|
|
328
|
+
return
|
|
329
|
+
if _active_context is self._write:
|
|
330
|
+
_active_context = None
|
|
331
|
+
try:
|
|
332
|
+
if self.relay is not None:
|
|
333
|
+
await self.relay.stop()
|
|
334
|
+
await drain()
|
|
335
|
+
finally:
|
|
336
|
+
try:
|
|
337
|
+
if self.pool is not None:
|
|
338
|
+
await _close_pool(self.pool)
|
|
339
|
+
finally:
|
|
340
|
+
self.pool = None
|
|
341
|
+
self.relay = None
|
|
342
|
+
self._write = None
|