unique_toolkit 2026.48.0.dev10__py3-none-any.whl → 2026.48.0.dev12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,88 @@
1
+ """Activity Records settings: size limits."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Final, Self
6
+
7
+ from pydantic import Field, model_validator
8
+ from pydantic_settings import BaseSettings, SettingsConfigDict
9
+
10
+ from unique_toolkit.experimental.resources.activity_records.json_size import (
11
+ json_byte_length,
12
+ )
13
+ from unique_toolkit.experimental.resources.activity_records.schemas import (
14
+ ACTIVITY_METADATA_MAX_LENGTH,
15
+ )
16
+
17
+ DEFAULT_MAX_DATA_SIZE_BYTES: Final = 1_048_576
18
+ DEFAULT_MAX_BATCH_SIZE_BYTES: Final = 8_388_608
19
+
20
+ _MAX_SAFE_INTEGER: Final = 2**53 - 1
21
+
22
+
23
+ _identifier = "x" * ACTIVITY_METADATA_MAX_LENGTH["identifier"]
24
+ _text = "x" * ACTIVITY_METADATA_MAX_LENGTH["text"]
25
+
26
+ # Worst case ``data``: every metadata field at its cap and ``criticalData`` shrunk to its shell.
27
+ MIN_MAX_DATA_SIZE_BYTES: Final = json_byte_length(
28
+ {
29
+ "actor": {
30
+ "id": _identifier,
31
+ "type": "service",
32
+ "ip": "x" * ACTIVITY_METADATA_MAX_LENGTH["ip"],
33
+ },
34
+ "agent": {"id": _identifier},
35
+ "companyId": _identifier,
36
+ "targetType": _identifier,
37
+ "targetId": _identifier,
38
+ "targetName": _text,
39
+ "subtype": _text,
40
+ "correlationId": _identifier,
41
+ "criticalData": {
42
+ "truncated": True,
43
+ "originalSizeBytesAtLeast": _MAX_SAFE_INTEGER,
44
+ "maxDataSizeBytes": _MAX_SAFE_INTEGER,
45
+ "value": None,
46
+ },
47
+ }
48
+ )
49
+
50
+ # Envelope fields outside ``data`` plus punctuation. ``source`` and ``type`` are uncapped,
51
+ # so this is a generous allowance, not an exact one.
52
+ EVENT_ENVELOPE_RESERVE_BYTES: Final = (
53
+ json_byte_length(
54
+ {
55
+ "specversion": "1.0",
56
+ "id": "00000000-0000-0000-0000-000000000000",
57
+ "source": "x" * 128,
58
+ "type": "x" * 128,
59
+ "time": "2026-01-01T00:00:00.000Z",
60
+ "datacontenttype": "application/json",
61
+ }
62
+ )
63
+ + 2
64
+ )
65
+
66
+
67
+ class ActivityRecordsConfig(BaseSettings):
68
+ """Read from ``ACTIVITY_RECORDS_MAX_DATA_SIZE_BYTES`` and ``ACTIVITY_RECORDS_MAX_BATCH_SIZE_BYTES``."""
69
+
70
+ model_config = SettingsConfigDict(
71
+ env_prefix="ACTIVITY_RECORDS_", extra="ignore", hide_input_in_errors=True
72
+ )
73
+
74
+ max_data_size_bytes: int = Field(
75
+ default=DEFAULT_MAX_DATA_SIZE_BYTES, ge=MIN_MAX_DATA_SIZE_BYTES
76
+ )
77
+ max_batch_size_bytes: int = DEFAULT_MAX_BATCH_SIZE_BYTES
78
+
79
+ @model_validator(mode="after")
80
+ def _one_event_fits_a_batch(self) -> Self:
81
+ minimum = self.max_data_size_bytes + EVENT_ENVELOPE_RESERVE_BYTES
82
+ if self.max_batch_size_bytes < minimum:
83
+ raise ValueError(
84
+ f"max_batch_size_bytes ({self.max_batch_size_bytes}) must be at least "
85
+ f"max_data_size_bytes plus the envelope reserve ({minimum}), so one "
86
+ "maximally-sized event always fits a batch by itself"
87
+ )
88
+ return self
@@ -0,0 +1,263 @@
1
+ """Byte-exact JSON sizing and truncation. Sizes are those of ``json_bytes``, except numbers,
2
+ which count as Postgres jsonb prints them: the relay reads payloads back with ``json.loads``."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ import math
8
+ from decimal import Decimal
9
+ from typing import Any, Final
10
+
11
+ from pydantic import JsonValue
12
+
13
+ # Bytes taken by the quotes and brackets around a value.
14
+ _QUOTE_BYTES: Final = 2
15
+ _CONTAINER_BRACKET_BYTES: Final = 2
16
+ _NULL_BYTES: Final = 4
17
+ # Lower than Node's 500: this walk recurses in Python, whose default limit is 1000 frames.
18
+ MAX_JSON_DEPTH: Final = 200
19
+
20
+
21
+ def _string_bytes(value: str) -> int:
22
+ # A lone surrogate counts 3 bytes, the size of the U+FFFD that replaces it before insert.
23
+ return len(json.dumps(value, ensure_ascii=False).encode("utf-8", "surrogatepass"))
24
+
25
+
26
+ def _null_non_finite(value: Any) -> Any:
27
+ if isinstance(value, float) and not math.isfinite(value):
28
+ return None
29
+ if isinstance(value, dict):
30
+ return {key: _null_non_finite(item) for key, item in value.items()}
31
+ if isinstance(value, (list, tuple)):
32
+ return [_null_non_finite(item) for item in value]
33
+ return value
34
+
35
+
36
+ def json_bytes(value: Any) -> bytes:
37
+ """Serialize compact UTF-8 JSON, writing NaN and Infinity as ``null``.
38
+
39
+ Float text can differ from ``JSON.stringify`` by a byte (``1e-07`` vs ``1e-7``), which
40
+ only matters within bytes of a size cap.
41
+ """
42
+ try:
43
+ text = json.dumps(
44
+ value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
45
+ )
46
+ except ValueError:
47
+ text = json.dumps(
48
+ _null_non_finite(value),
49
+ ensure_ascii=False,
50
+ separators=(",", ":"),
51
+ allow_nan=False,
52
+ )
53
+ return text.encode("utf-8")
54
+
55
+
56
+ def _number_bytes(value: int | float) -> int:
57
+ if isinstance(value, float) and not math.isfinite(value):
58
+ return _NULL_BYTES
59
+ text = json.dumps(value)
60
+ # Postgres jsonb prints numbers without an exponent, so the stored size is that of the digits.
61
+ return len(format(Decimal(text), "f")) if "e" in text else len(text)
62
+
63
+
64
+ def _scalar_bytes(value: Any) -> int:
65
+ if value is None:
66
+ return _NULL_BYTES
67
+ if isinstance(value, bool):
68
+ return 4 if value else 5
69
+ return _number_bytes(value)
70
+
71
+
72
+ def json_byte_length(value: Any) -> int:
73
+ """Size of the serialised text itself: a float with an exponent counts its short form."""
74
+ return len(json_bytes(value))
75
+
76
+
77
+ def json_byte_length_if_within_limit(
78
+ value: object, max_bytes: float, depth: int = 0
79
+ ) -> int | None:
80
+ """UTF-8 size of ``json_bytes(value)`` with numbers counted as jsonb prints them (a float
81
+ like ``1e21`` is 22, not 5), or ``None`` once it exceeds ``max_bytes`` (aborts early)."""
82
+ if max_bytes < 0 or depth > MAX_JSON_DEPTH:
83
+ return None
84
+ if value is None or isinstance(value, (bool, int, float)):
85
+ size = _scalar_bytes(value)
86
+ elif isinstance(value, str):
87
+ # Every character costs at least one byte, so a long string can fail without a scan.
88
+ if len(value) + _QUOTE_BYTES > max_bytes:
89
+ return None
90
+ size = _string_bytes(value)
91
+ elif isinstance(value, (list, tuple)):
92
+ return _estimate_array(value, max_bytes, depth)
93
+ elif isinstance(value, dict):
94
+ return _estimate_object(value, max_bytes, depth)
95
+ else:
96
+ raise TypeError(f"Not JSON serializable: {type(value).__name__}")
97
+ return size if size <= max_bytes else None
98
+
99
+
100
+ def _estimate_array(
101
+ value: list[Any] | tuple[Any, ...], max_bytes: float, depth: int
102
+ ) -> int | None:
103
+ if _CONTAINER_BRACKET_BYTES > max_bytes:
104
+ return None
105
+ size = _CONTAINER_BRACKET_BYTES
106
+ for index, item in enumerate(value):
107
+ separator = 1 if index > 0 else 0
108
+ item_bytes = json_byte_length_if_within_limit(
109
+ item, max_bytes - size - separator, depth + 1
110
+ )
111
+ if item_bytes is None:
112
+ return None
113
+ size += separator + item_bytes
114
+ return size
115
+
116
+
117
+ def _estimate_object(value: dict[str, Any], max_bytes: float, depth: int) -> int | None:
118
+ if _CONTAINER_BRACKET_BYTES > max_bytes:
119
+ return None
120
+ size = _CONTAINER_BRACKET_BYTES
121
+ for index, (key, field) in enumerate(value.items()):
122
+ separator = 1 if index > 0 else 0
123
+ key_bytes = _string_bytes(key)
124
+ remaining = max_bytes - size - separator - key_bytes - 1
125
+ field_bytes = json_byte_length_if_within_limit(field, remaining, depth + 1)
126
+ if field_bytes is None:
127
+ return None
128
+ size += separator + key_bytes + 1 + field_bytes
129
+ return size
130
+
131
+
132
+ def _truncate_string(value: str, max_bytes: float) -> tuple[str, int]:
133
+ """Longest prefix whose JSON size fits ``max_bytes`` (``""`` when even that does not)."""
134
+ low, high = 0, min(len(value), max(int(max_bytes), 0))
135
+ while low < high:
136
+ middle = (low + high + 1) // 2
137
+ if json_byte_length_if_within_limit(value[:middle], max_bytes) is None:
138
+ high = middle - 1
139
+ else:
140
+ low = middle
141
+ return value[:low], _string_bytes(value[:low])
142
+
143
+
144
+ def _shrink(value: object, max_bytes: float, depth: int = 0) -> tuple[Any, int]:
145
+ if depth > MAX_JSON_DEPTH:
146
+ return None, _NULL_BYTES
147
+ if not isinstance(value, (str, list, tuple, dict)):
148
+ return value, _scalar_bytes(value)
149
+ whole = json_byte_length_if_within_limit(value, max_bytes, depth)
150
+ if whole is not None:
151
+ return value, whole
152
+ if isinstance(value, str):
153
+ return _truncate_string(value, max_bytes)
154
+ if isinstance(value, dict):
155
+ return _truncate_object(value, max_bytes, depth)
156
+ return _truncate_array(value, max_bytes, depth)
157
+
158
+
159
+ def _truncate_array(
160
+ value: list[Any] | tuple[Any, ...], max_bytes: float, depth: int
161
+ ) -> tuple[list[Any], int]:
162
+ """Keeps the leading items that fit, shrinks the boundary item, drops the rest."""
163
+ kept: list[Any] = []
164
+ size = _CONTAINER_BRACKET_BYTES
165
+ for item in value:
166
+ separator = 1 if kept else 0
167
+ item_bytes = json_byte_length_if_within_limit(
168
+ item, max_bytes - size - separator, depth + 1
169
+ )
170
+ if item_bytes is not None:
171
+ kept.append(item)
172
+ size += separator + item_bytes
173
+ continue
174
+ budget = max_bytes - size - separator
175
+ if budget > 0:
176
+ shrunk, shrunk_bytes = _shrink(item, budget, depth + 1)
177
+ if size + separator + shrunk_bytes <= max_bytes:
178
+ kept.append(shrunk)
179
+ size += separator + shrunk_bytes
180
+ break
181
+ return kept, size
182
+
183
+
184
+ def _truncate_object(
185
+ value: dict[str, Any], max_bytes: float, depth: int
186
+ ) -> tuple[dict[str, Any], int]:
187
+ """Shrinks or drops the largest fields first until the object fits."""
188
+ fields = []
189
+ for key, field in value.items():
190
+ estimated = json_byte_length_if_within_limit(field, max_bytes, depth + 1)
191
+ fields.append(
192
+ (
193
+ key,
194
+ field,
195
+ _string_bytes(key),
196
+ max_bytes + 1 if estimated is None else estimated,
197
+ )
198
+ )
199
+
200
+ size = _CONTAINER_BRACKET_BYTES + sum(
201
+ key_bytes + 1 + field_bytes for _, _, key_bytes, field_bytes in fields
202
+ )
203
+ size += max(len(fields) - 1, 0)
204
+ remaining_keys = len(fields)
205
+ shrunk_fields: dict[str, Any] = {}
206
+ dropped: set[str] = set()
207
+
208
+ for key, field, key_bytes, field_bytes in sorted(fields, key=lambda f: -f[3]):
209
+ if size <= max_bytes:
210
+ break
211
+ removed_comma = 1 if remaining_keys > 1 else 0
212
+ remaining_keys -= 1
213
+ size -= key_bytes + 1 + field_bytes + removed_comma
214
+ key_overhead = key_bytes + 1 + (1 if remaining_keys > 0 else 0)
215
+ budget = max_bytes - size - key_overhead
216
+ if budget > 0:
217
+ shrunk, shrunk_bytes = _shrink(field, budget, depth + 1)
218
+ if size + key_overhead + shrunk_bytes <= max_bytes:
219
+ shrunk_fields[key] = shrunk
220
+ size += key_overhead + shrunk_bytes
221
+ remaining_keys += 1
222
+ continue
223
+ dropped.add(key)
224
+
225
+ result = {
226
+ key: shrunk_fields.get(key, field)
227
+ for key, field in value.items()
228
+ if key not in dropped
229
+ }
230
+ return result, size
231
+
232
+
233
+ def cap_depth(value: object, depth: int = 0) -> tuple[Any, bool]:
234
+ """Replaces every container nested at ``MAX_JSON_DEPTH`` or deeper with ``None``, which
235
+ keeps the result inside the sizing walk's depth limit. Also says whether it cut anything."""
236
+ if not isinstance(value, (list, tuple, dict)):
237
+ return value, False
238
+ if depth >= MAX_JSON_DEPTH:
239
+ return None, True
240
+ if isinstance(value, dict):
241
+ capped = {key: cap_depth(item, depth + 1) for key, item in value.items()}
242
+ return (
243
+ {key: item for key, (item, _) in capped.items()},
244
+ any(cut for _, cut in capped.values()),
245
+ )
246
+ capped_items = [cap_depth(item, depth + 1) for item in value]
247
+ return [item for item, _ in capped_items], any(cut for _, cut in capped_items)
248
+
249
+
250
+ def truncate_json_value_to_bytes(value: object, max_bytes: float) -> JsonValue:
251
+ """Shrinks ``value`` so its compact JSON fits ``max_bytes`` (best effort below 2 bytes):
252
+ strings are cut, arrays keep leading items, objects shrink or drop their largest fields."""
253
+ return _shrink(value, max(max_bytes, 0))[0]
254
+
255
+
256
+ def truncate_to_utf16_length(value: str, max_length: int) -> str:
257
+ """Cuts ``value`` to at most ``max_length`` UTF-16 code units, never inside a surrogate pair."""
258
+ units = 0
259
+ for index, char in enumerate(value):
260
+ units += 2 if char > "￿" else 1
261
+ if units > max_length:
262
+ return value[:index]
263
+ return value
@@ -0,0 +1,294 @@
1
+ """Prepare Activity Records outbox rows from camelCase wire dictionaries."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import contextlib
6
+ import logging
7
+ import math
8
+ import re
9
+ from collections.abc import Callable, Sequence
10
+ from dataclasses import dataclass
11
+ from typing import Any, Final, overload
12
+
13
+ from unique_toolkit.experimental.resources.activity_records.config import (
14
+ DEFAULT_MAX_DATA_SIZE_BYTES,
15
+ )
16
+ from unique_toolkit.experimental.resources.activity_records.json_size import (
17
+ cap_depth,
18
+ json_byte_length,
19
+ json_byte_length_if_within_limit,
20
+ truncate_json_value_to_bytes,
21
+ )
22
+ from unique_toolkit.experimental.resources.activity_records.schemas import (
23
+ ActivityCloudEvent,
24
+ )
25
+
26
+ try:
27
+ from opentelemetry import trace as otel_trace
28
+ except ImportError: # opentelemetry-api is optional
29
+ otel_trace = None
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+ WireEvent = dict[str, Any]
34
+
35
+ _EMPTY_BATCH_BYTES: Final = 2
36
+ _UNSAFE_CHARACTERS = re.compile("[\x00\ud800-\udfff]")
37
+ _REPLACEMENT_CHARACTER: Final = "�"
38
+
39
+
40
+ @dataclass(frozen=True, slots=True)
41
+ class OutboxRow:
42
+ id: str
43
+ company_id: str
44
+ source: str
45
+ payload: WireEvent
46
+
47
+
48
+ @dataclass(frozen=True, slots=True)
49
+ class FitResult:
50
+ event: WireEvent
51
+ critical_data_truncated: bool
52
+
53
+
54
+ @dataclass(frozen=True, slots=True)
55
+ class FitEventsResult:
56
+ events: list[WireEvent]
57
+ # Events whose criticalData was shrunk, or that are still over the cap.
58
+ truncated_events: list[WireEvent]
59
+
60
+
61
+ @dataclass(frozen=True, slots=True)
62
+ class PreparedRows:
63
+ rows: list[OutboxRow]
64
+ truncated_events: list[WireEvent]
65
+
66
+
67
+ def _shrink_critical_data(
68
+ event: WireEvent,
69
+ max_data_size_bytes: int,
70
+ original_size_at_least: int | None = None,
71
+ ) -> tuple[WireEvent, bool]:
72
+ data = event["data"]
73
+ if "criticalData" not in data:
74
+ return event, False
75
+
76
+ # A lower bound, not a measurement: the oversized data was never fully sized.
77
+ if original_size_at_least is None:
78
+ original_size_at_least = max_data_size_bytes + 1
79
+ without_critical_data = {
80
+ key: value for key, value in data.items() if key != "criticalData"
81
+ }
82
+ key_overhead = (
83
+ json_byte_length("criticalData") + 1 + (1 if without_critical_data else 0)
84
+ )
85
+ size_without = json_byte_length_if_within_limit(
86
+ without_critical_data, max_data_size_bytes
87
+ )
88
+ wrapper_shell = json_byte_length(
89
+ {
90
+ "truncated": True,
91
+ "originalSizeBytesAtLeast": original_size_at_least,
92
+ "maxDataSizeBytes": max_data_size_bytes,
93
+ "value": None,
94
+ }
95
+ ) - json_byte_length(None)
96
+
97
+ budget = (
98
+ -1
99
+ if size_without is None
100
+ else max_data_size_bytes - size_without - key_overhead - wrapper_shell
101
+ )
102
+ value = (
103
+ truncate_json_value_to_bytes(data["criticalData"], budget)
104
+ if budget > 0
105
+ else None
106
+ )
107
+ critical_data = {
108
+ "truncated": True,
109
+ "originalSizeBytesAtLeast": original_size_at_least,
110
+ "maxDataSizeBytes": max_data_size_bytes,
111
+ "value": value,
112
+ }
113
+ return {
114
+ **event,
115
+ "data": {**without_critical_data, "criticalData": critical_data},
116
+ }, True
117
+
118
+
119
+ def _cap_critical_data_depth(event: WireEvent) -> tuple[WireEvent, bool]:
120
+ # criticalData sits one level inside data, and its value one more once the truncation wrapper
121
+ # holds it, so the cap starts at depth 2 and the wrapped value stays within the walk's limit.
122
+ data = event["data"]
123
+ if "criticalData" not in data:
124
+ return event, False
125
+ capped, cut = cap_depth(data["criticalData"], 2)
126
+ if not cut:
127
+ return event, False
128
+ return {**event, "data": {**data, "criticalData": capped}}, True
129
+
130
+
131
+ def fit_event_to_limit(
132
+ event: WireEvent, max_data_size_bytes: int = DEFAULT_MAX_DATA_SIZE_BYTES
133
+ ) -> FitResult:
134
+ """Fits ``event`` into ``max_data_size_bytes`` best effort and never drops it:
135
+ ``criticalData`` shrinks into the bytes left once the rest of ``data`` is counted."""
136
+ capped, depth_cut = _cap_critical_data_depth(event)
137
+ if (
138
+ not depth_cut
139
+ and json_byte_length_if_within_limit(event["data"], max_data_size_bytes)
140
+ is not None
141
+ ):
142
+ return FitResult(event, critical_data_truncated=False)
143
+ # A depth cut gets the same marker as a size cut; its original size is measured, not a bound.
144
+ original_size = None
145
+ if depth_cut:
146
+ with contextlib.suppress(RecursionError):
147
+ original_size = json_byte_length(with_postgres_safe_strings(event["data"]))
148
+ fitted, shrunk = _shrink_critical_data(capped, max_data_size_bytes, original_size)
149
+ return FitResult(fitted, critical_data_truncated=shrunk)
150
+
151
+
152
+ def is_oversized(event: WireEvent, max_data_size_bytes: int) -> bool:
153
+ return json_byte_length_if_within_limit(event["data"], max_data_size_bytes) is None
154
+
155
+
156
+ def fit_events_to_limit(
157
+ events: Sequence[WireEvent], max_data_size_bytes: int
158
+ ) -> FitEventsResult:
159
+ fitted_events: list[WireEvent] = []
160
+ truncated_events: list[WireEvent] = []
161
+ for event in events:
162
+ fit = fit_event_to_limit(event, max_data_size_bytes)
163
+ if fit.critical_data_truncated:
164
+ logger.warning(
165
+ "Activity event criticalData truncated to fit the size or depth limit: "
166
+ "eventType=%s source=%s maxDataSizeBytes=%s",
167
+ event["type"],
168
+ event["source"],
169
+ max_data_size_bytes,
170
+ )
171
+ still_oversized = is_oversized(fit.event, max_data_size_bytes)
172
+ if still_oversized and not fit.critical_data_truncated:
173
+ logger.warning(
174
+ "Activity event data still exceeds the size limit after best-effort truncation: "
175
+ "eventType=%s source=%s maxDataSizeBytes=%s",
176
+ event["type"],
177
+ event["source"],
178
+ max_data_size_bytes,
179
+ )
180
+ fitted_events.append(fit.event)
181
+ if fit.critical_data_truncated or still_oversized:
182
+ truncated_events.append(fit.event)
183
+ return FitEventsResult(fitted_events, truncated_events)
184
+
185
+
186
+ def _active_trace_id() -> str | None:
187
+ if otel_trace is None:
188
+ return None
189
+ span_context = otel_trace.get_current_span().get_span_context()
190
+ return format(span_context.trace_id, "032x") if span_context.is_valid else None
191
+
192
+
193
+ def with_correlation_id(events: Sequence[WireEvent]) -> list[WireEvent]:
194
+ """Stamps the active trace id on events without one. Must run in the caller's context."""
195
+ correlation_id = _active_trace_id()
196
+ if correlation_id is None:
197
+ return list(events)
198
+ return [
199
+ event
200
+ if event["data"].get("correlationId")
201
+ else {**event, "data": {**event["data"], "correlationId": correlation_id}}
202
+ for event in events
203
+ ]
204
+
205
+
206
+ @overload
207
+ def with_postgres_safe_strings(value: str) -> str: ...
208
+ @overload
209
+ def with_postgres_safe_strings(value: WireEvent) -> WireEvent: ...
210
+ def with_postgres_safe_strings(value: Any) -> Any:
211
+ """Replaces U+0000 and surrogate code points in every string and key: Postgres ``jsonb``
212
+ and ``text`` reject them, and one bad event fails the whole insert it shares."""
213
+ if isinstance(value, str):
214
+ return _UNSAFE_CHARACTERS.sub(_REPLACEMENT_CHARACTER, value)
215
+ if isinstance(value, list):
216
+ return [with_postgres_safe_strings(item) for item in value]
217
+ if isinstance(value, dict):
218
+ return {
219
+ with_postgres_safe_strings(key): with_postgres_safe_strings(item)
220
+ for key, item in value.items()
221
+ }
222
+ return value
223
+
224
+
225
+ def prepare_rows(
226
+ company_id: str,
227
+ events: Sequence[ActivityCloudEvent],
228
+ max_data_size_bytes: int = DEFAULT_MAX_DATA_SIZE_BYTES,
229
+ ) -> PreparedRows:
230
+ """Outbox rows for ``events``. Call in the caller's context: the correlation id comes
231
+ from the active OpenTelemetry trace."""
232
+ if with_postgres_safe_strings(company_id) != company_id:
233
+ raise ValueError("company_id contains characters Postgres rejects")
234
+ if any(event.data.company_id != company_id for event in events):
235
+ raise ValueError("An event belongs to another company than company_id")
236
+ fitted = fit_events_to_limit(
237
+ with_correlation_id([event.to_wire() for event in events]), max_data_size_bytes
238
+ )
239
+ rows = []
240
+ for event in fitted.events:
241
+ payload = with_postgres_safe_strings(event)
242
+ rows.append(OutboxRow(payload["id"], company_id, payload["source"], payload))
243
+ return PreparedRows(rows, fitted.truncated_events)
244
+
245
+
246
+ def split_into_batches_under_limit[T](
247
+ items: Sequence[T],
248
+ event_of: Callable[[T], WireEvent],
249
+ max_batch_size_bytes: int,
250
+ max_data_size_bytes: int,
251
+ ) -> list[list[T]]:
252
+ """Groups ``items`` into as few messages as fit under ``max_batch_size_bytes``, in order.
253
+ An oversized event, or one over the batch cap on its own, is sent alone, never dropped."""
254
+ batches: list[list[T]] = []
255
+ current: list[T] = []
256
+ current_bytes: float = _EMPTY_BATCH_BYTES
257
+
258
+ for item in items:
259
+ event = event_of(item)
260
+ if is_oversized(event, max_data_size_bytes):
261
+ if current:
262
+ batches.append(current)
263
+ batches.append([item])
264
+ current, current_bytes = [], _EMPTY_BATCH_BYTES
265
+ continue
266
+
267
+ event_bytes = json_byte_length_if_within_limit(event, max_batch_size_bytes)
268
+ separator = 1 if current else 0
269
+ if (
270
+ event_bytes is not None
271
+ and current_bytes + separator + event_bytes <= max_batch_size_bytes
272
+ ):
273
+ current.append(item)
274
+ current_bytes += separator + event_bytes
275
+ continue
276
+
277
+ if current:
278
+ batches.append(current)
279
+ if event_bytes is None:
280
+ logger.warning(
281
+ "Activity event exceeds the whole-batch size limit even alone; sending it in "
282
+ "its own message anyway: eventType=%s source=%s maxBatchSizeBytes=%s",
283
+ event["type"],
284
+ event["source"],
285
+ max_batch_size_bytes,
286
+ )
287
+ current = [item]
288
+ current_bytes = (
289
+ math.inf if event_bytes is None else _EMPTY_BATCH_BYTES + event_bytes
290
+ )
291
+
292
+ if current:
293
+ batches.append(current)
294
+ return batches
@@ -2,10 +2,12 @@
2
2
  rejects a whole batch if one event fails. Extra keys are forbidden on every path. The
3
3
  constructor and plain ``model_validate`` accept both the camelCase wire keys and the
4
4
  Python field names (the toolkit convention); ``ActivityCloudEvent.from_wire`` accepts only
5
- the wire keys, like zod. Experimental: the shape may still change."""
5
+ the wire keys, like zod. ``ActivityCloudEvent.build`` can shorten over-length metadata.
6
+ Experimental: the shape may still change."""
6
7
 
7
8
  from __future__ import annotations
8
9
 
10
+ import re
9
11
  from collections.abc import Mapping
10
12
  from datetime import UTC, datetime
11
13
  from typing import Annotated, Any, Final, Literal, Self
@@ -17,16 +19,26 @@ from pydantic import (
17
19
  BaseModel,
18
20
  Field,
19
21
  JsonValue,
22
+ PrivateAttr,
20
23
  StringConstraints,
24
+ ValidationInfo,
25
+ ValidatorFunctionWrapHandler,
26
+ WrapValidator,
21
27
  field_serializer,
22
28
  model_validator,
23
29
  )
30
+ from pydantic.alias_generators import to_camel
24
31
 
25
32
  from unique_toolkit._common.pydantic_helpers import get_configuration_dict
33
+ from unique_toolkit.experimental.resources.activity_records.json_size import (
34
+ truncate_to_utf16_length,
35
+ )
26
36
 
27
37
  ACTIVITY_METADATA_MAX_LENGTH: Final = {"identifier": 128, "ip": 64, "text": 256}
28
38
 
29
39
  _model_config = get_configuration_dict(extra="forbid")
40
+ # Same replacement as prepare.py: pydantic and Postgres both reject a lone surrogate.
41
+ _LONE_SURROGATE = re.compile("[\ud800-\udfff]")
30
42
  _NIL_UUID: Final = UUID(int=0)
31
43
  _MAX_UUID: Final = UUID(int=2**128 - 1)
32
44
 
@@ -38,22 +50,44 @@ def _is_zod_uuid(value: UUID) -> UUID:
38
50
  raise ValueError("must be a UUID of version 1 to 8")
39
51
 
40
52
 
41
- def _max_length(limit: int) -> AfterValidator:
42
- # zod counts UTF-16 code units, Python counts code points.
43
- def check(value: str) -> str:
44
- if len(value.encode("utf-16-le", "surrogatepass")) > 2 * limit:
53
+ def _max_length(limit: int, path: str | None = None) -> WrapValidator:
54
+ # zod counts UTF-16 code units, Python counts code points. With ``truncate`` set in the
55
+ # validation context, lone surrogates are replaced and over-length values cut and recorded.
56
+ def check(
57
+ value: Any, handler: ValidatorFunctionWrapHandler, info: ValidationInfo
58
+ ) -> str:
59
+ context = info.context or {}
60
+ if context.get("truncate") and isinstance(value, str):
61
+ value = _LONE_SURROGATE.sub("\ufffd", value)
62
+ cut = truncate_to_utf16_length(value, limit)
63
+ if cut != value:
64
+ # Wire path like Node (actor.id, targetName); a bare field name when unset.
65
+ context.get("truncated", []).append(
66
+ path or to_camel(info.field_name or "")
67
+ )
68
+ value = cut
69
+ result = handler(value)
70
+ if len(result.encode("utf-16-le", "surrogatepass")) > 2 * limit:
45
71
  raise ValueError(f"must be at most {limit} characters")
46
- return value
72
+ return result
47
73
 
48
- return AfterValidator(check)
74
+ return WrapValidator(check)
49
75
 
50
76
 
51
77
  _NonEmpty = Annotated[str, StringConstraints(strict=True, min_length=1)]
52
78
  _Identifier = Annotated[
53
79
  _NonEmpty, _max_length(ACTIVITY_METADATA_MAX_LENGTH["identifier"])
54
80
  ]
55
- _Ip = Annotated[_NonEmpty, _max_length(ACTIVITY_METADATA_MAX_LENGTH["ip"])]
56
81
  _Text = Annotated[_NonEmpty, _max_length(ACTIVITY_METADATA_MAX_LENGTH["text"])]
82
+ _ActorId = Annotated[
83
+ _NonEmpty, _max_length(ACTIVITY_METADATA_MAX_LENGTH["identifier"], "actor.id")
84
+ ]
85
+ _ActorIp = Annotated[
86
+ _NonEmpty, _max_length(ACTIVITY_METADATA_MAX_LENGTH["ip"], "actor.ip")
87
+ ]
88
+ _AgentId = Annotated[
89
+ _NonEmpty, _max_length(ACTIVITY_METADATA_MAX_LENGTH["identifier"], "agent.id")
90
+ ]
57
91
 
58
92
 
59
93
  class ActivityActor(BaseModel):
@@ -61,9 +95,9 @@ class ActivityActor(BaseModel):
61
95
 
62
96
  model_config = _model_config
63
97
 
64
- id: _Identifier
98
+ id: _ActorId
65
99
  type: Literal["user", "service"]
66
- ip: _Ip | None = None
100
+ ip: _ActorIp | None = None
67
101
 
68
102
 
69
103
  class ActivityAgent(BaseModel):
@@ -71,7 +105,7 @@ class ActivityAgent(BaseModel):
71
105
 
72
106
  model_config = _model_config
73
107
 
74
- id: _Identifier
108
+ id: _AgentId
75
109
 
76
110
 
77
111
  class ActivityEventData(BaseModel):
@@ -112,6 +146,23 @@ class ActivityCloudEvent(BaseModel):
112
146
  time: AwareDatetime = Field(default_factory=lambda: datetime.now(UTC))
113
147
  datacontenttype: Literal["application/json"] | None = "application/json"
114
148
  data: ActivityEventData
149
+ _truncated_fields: list[str] = PrivateAttr(default_factory=list)
150
+
151
+ @classmethod
152
+ def build(cls, **fields: Any) -> Self:
153
+ """Validate ``fields`` (``data`` as a dict) like the constructor, but shorten
154
+ over-length metadata instead of rejecting it. See ``truncated_fields``."""
155
+ truncated: list[str] = []
156
+ event = cls.model_validate(
157
+ fields, context={"truncate": True, "truncated": truncated}
158
+ )
159
+ event._truncated_fields = truncated
160
+ return event
161
+
162
+ @property
163
+ def truncated_fields(self) -> list[str]:
164
+ """Names of the metadata fields ``build`` shortened; empty otherwise."""
165
+ return self._truncated_fields
115
166
 
116
167
  @field_serializer("time")
117
168
  def _serialize_time(self, value: datetime) -> str:
@@ -119,6 +119,7 @@ class LanguageModelName(StrEnum):
119
119
  GEMINI_3_PRO_PREVIEW = "litellm:gemini-3-pro-preview"
120
120
  GEMINI_4_ARGON = "litellm:gemini-4-argon"
121
121
  GEMINI_4_CARBON = "litellm:gemini-4-carbon"
122
+ GEMINI_4_HYDROGEN = "litellm:gemini-4-hydrogen"
122
123
  GROK_4_1_FAST_NON_REASONING = "litellm:grok-4-1-fast-non-reasoning"
123
124
  GROK_4_1_FAST_REASONING = "litellm:grok-4-1-fast-reasoning"
124
125
  GROK_4_5 = "litellm:grok-4-5"
@@ -182,6 +183,7 @@ class LanguageModelName(StrEnum):
182
183
  VERTEX_GEMINI_3_8_FLASH = "litellm:vertex-gemini-3-8-flash"
183
184
  VERTEX_GEMINI_4_ARGON = "litellm:vertex-gemini-4-argon"
184
185
  VERTEX_GEMINI_4_CARBON = "litellm:vertex-gemini-4-carbon"
186
+ VERTEX_GEMINI_4_HYDROGEN = "litellm:vertex-gemini-4-hydrogen"
185
187
 
186
188
 
187
189
  class EncoderName(StrEnum):
@@ -2577,6 +2579,35 @@ class LanguageModelInfo(BaseModel):
2577
2579
  default_options={"reasoning_effort": "medium"},
2578
2580
  supported_reasoning_efforts=["low", "medium", "high"],
2579
2581
  )
2582
+ case (
2583
+ LanguageModelName.GEMINI_4_HYDROGEN
2584
+ | LanguageModelName.VERTEX_GEMINI_4_HYDROGEN
2585
+ ):
2586
+ return cls(
2587
+ name=model_name,
2588
+ capabilities=[
2589
+ ModelCapabilities.FUNCTION_CALLING,
2590
+ ModelCapabilities.STREAMING,
2591
+ ModelCapabilities.VISION,
2592
+ ModelCapabilities.STRUCTURED_OUTPUT,
2593
+ ModelCapabilities.REASONING,
2594
+ ],
2595
+ provider=LanguageModelProvider.LITELLM,
2596
+ family=ModelFamily.GOOGLE,
2597
+ version="gemini-4-hydrogen",
2598
+ encoder_name=EncoderName.O200K_BASE, # TODO: Update encoder with litellm
2599
+ token_limits=LanguageModelTokenLimits(
2600
+ # Unified 1,048,576 context window; 65,536 single-request output cap.
2601
+ # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/4-hydrogen
2602
+ token_limit_input=1_048_576,
2603
+ token_limit_output=65_536,
2604
+ ),
2605
+ published_at=date(2026, 11, 12),
2606
+ # Google also has xhigh, but LiteLLM can't send it via reasoning_effort yet.
2607
+ # https://docs.litellm.ai/blog/gemini-4-hydrogen
2608
+ default_options={"reasoning_effort": "medium"},
2609
+ supported_reasoning_efforts=["low", "medium", "high"],
2610
+ )
2580
2611
  case LanguageModelName.GROK_4_1_FAST_NON_REASONING:
2581
2612
  return cls(
2582
2613
  name=model_name,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: unique-toolkit
3
- Version: 2026.48.0.dev10
3
+ Version: 2026.48.0.dev12
4
4
  Summary:
5
5
  Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
6
6
  Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
16
16
  Requires-Dist: openai>=1.105.0,<3
17
17
  Requires-Dist: platformdirs>=4.0.0,<5
18
18
  Requires-Dist: pillow>=13.0.0
19
- Requires-Dist: unique-sdk>=2026.48.0.dev8,<2026.48.0rc0
19
+ Requires-Dist: unique-sdk>=2026.48.0.dev12,<2026.48.0rc0
20
20
  Requires-Dist: jambo>=0.1.2,<0.2
21
21
  Requires-Dist: jinja2>=3.1.6,<4
22
22
  Requires-Dist: docxtpl>=0.20.1,<0.21
@@ -362,7 +362,10 @@ unique_toolkit/experimental/integrations/openai/streaming/event_routing/subscrib
362
362
  unique_toolkit/experimental/integrations/openai/streaming/event_routing/subscribers/progress_log_persister.py,sha256=VCYGZY3WSpI7mvVs2gO89-M0JenakW01BGfBfFkPDrw,3499
363
363
  unique_toolkit/experimental/resources/__init__.py,sha256=KdMPaaneY0twZ-JBfwImRjXZaWT68NVuOIeqdNXFh_Y,3377
364
364
  unique_toolkit/experimental/resources/activity_records/__init__.py,sha256=QEvl-xjtN-5oQT6v1uPlKbnfJs5EHb4f7EDhOg6UyRU,375
365
- unique_toolkit/experimental/resources/activity_records/schemas.py,sha256=l0AJLOR1Tj2ukRH3Kw06mVVFKEouHn6w05XToK47Go8,4728
365
+ unique_toolkit/experimental/resources/activity_records/config.py,sha256=R-yKToKpU4w5oHmqQMhFTadEL925DhAhunDsubQnRpY,2912
366
+ unique_toolkit/experimental/resources/activity_records/json_size.py,sha256=gfIRmI08txf0Y_wEyPTqtYrUk9FHwSouYC4R7qtdgdY,9761
367
+ unique_toolkit/experimental/resources/activity_records/prepare.py,sha256=PKezBDYus3M6qZluI1Nh-oTpTjU44V37r9lN9dzILNc,10269
368
+ unique_toolkit/experimental/resources/activity_records/schemas.py,sha256=6-jbRHJrqOjbPp9ij1etpbZjKZ6ol-TiE60vdiDkqJA,6864
366
369
  unique_toolkit/experimental/resources/content_folder/__init__.py,sha256=c5MUj1xIP10L7tcM4-hG2m1_yUMVfKr_jHAncUXVNvo,2492
367
370
  unique_toolkit/experimental/resources/content_folder/functions.py,sha256=nJle9Qnh-x1SLNetBBhTbQPu-zEBfQaHFnMq8N9RGEQ,16692
368
371
  unique_toolkit/experimental/resources/content_folder/schemas.py,sha256=LYWRpMLFVg_iKT9dw2AyCNfTPVmKeB8erMSsgbIkZ48,5345
@@ -406,7 +409,7 @@ unique_toolkit/language_model/constants.py,sha256=B-topqW0r83dkC_25DeQfnPk3n53qz
406
409
  unique_toolkit/language_model/default_language_model.py,sha256=mcyuFs_V6tp52C8xJSHiEHO4kFGj6xed1gvozvCRD7k,1181
407
410
  unique_toolkit/language_model/errors.py,sha256=0nm4psGCGJM2B7LIyKXy7WuswRzwARIgIK5xoC4xo34,3186
408
411
  unique_toolkit/language_model/functions.py,sha256=6Le6dDm9su6xk3sQ8IPiyUn2tf1qvzxHVUhyTTgP8Kw,28109
409
- unique_toolkit/language_model/infos.py,sha256=nWm0X3Kjeyam4EfA6ALbuJ8wYxvA5zL8_Z89JYniMf8,189850
412
+ unique_toolkit/language_model/infos.py,sha256=ZXEPfmc--icQhH13-UJbm-89ojKCgS43ULd1VhvQheo,191539
410
413
  unique_toolkit/language_model/invocation_stats.py,sha256=CQbpUz394IsVd8eNTB1gPSMt5f0UXmkN1amNsqtAW6U,5857
411
414
  unique_toolkit/language_model/model_costs.py,sha256=AVHAuMTzsglzusWsVF_xwBVEndST-TVCVi0H49OhdvE,7419
412
415
  unique_toolkit/language_model/prompt.py,sha256=JSawaLjQg3VR-E2fK8engFyJnNdk21zaO8pPIodzN4Q,3991
@@ -437,6 +440,6 @@ unique_toolkit/short_term_memory/service.py,sha256=2JMBEBn6GoyK-a0xvdcWpPft-3C74
437
440
  unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
438
441
  unique_toolkit/smart_rules/compile.py,sha256=vRe0aMJWuVIK-VsqomgxQImzke7MDmwgSxxJfYCa7zw,1386
439
442
  unique_toolkit/test_utilities/events.py,sha256=FLrhMh0A_mQutAUjoD0FdMcVQ4Gq31r35OVGzRjvLro,6760
440
- unique_toolkit-2026.48.0.dev10.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
441
- unique_toolkit-2026.48.0.dev10.dist-info/METADATA,sha256=KEfKeV7yvL6YZmLfI0Gd83eiPdnGfyUj_MIsLh5TgHQ,7822
442
- unique_toolkit-2026.48.0.dev10.dist-info/RECORD,,
443
+ unique_toolkit-2026.48.0.dev12.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
444
+ unique_toolkit-2026.48.0.dev12.dist-info/METADATA,sha256=KTe7yhRozXBOnHPFaIHK98eb15KVYyisASN58FNmKQk,7823
445
+ unique_toolkit-2026.48.0.dev12.dist-info/RECORD,,