numbunch 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
numbunch/__init__.py ADDED
@@ -0,0 +1,37 @@
1
+ """Numbunch Python SDK (MIT).
2
+
3
+ from numbunch import Numbunch
4
+
5
+ numbunch = Numbunch("nb_pub_…", host="https://in.example.com", secret_key="nb_sec_…")
6
+ numbunch.capture("user@example.com", "Signed up", {"plan": "pro"})
7
+ variant = numbunch.get_feature_flag("landing-home-new", "user@example.com")
8
+ numbunch.shutdown()
9
+ """
10
+
11
+ from .client import DEFAULT_HOST, Numbunch
12
+ from .context import parse_baggage, request_context, trace_properties, with_request_context
13
+ from .evaluate import Evaluation, evaluate
14
+ from .hashing import murmur3_v1, murmur3_x86_32, sha1_v1
15
+ from .matching import Inconclusive
16
+ from .uuid7 import uuid7
17
+ from .version import VERSION
18
+
19
+ __version__ = VERSION
20
+
21
+ __all__ = [
22
+ "DEFAULT_HOST",
23
+ "VERSION",
24
+ "Evaluation",
25
+ "Inconclusive",
26
+ "Numbunch",
27
+ "__version__",
28
+ "evaluate",
29
+ "murmur3_v1",
30
+ "murmur3_x86_32",
31
+ "parse_baggage",
32
+ "request_context",
33
+ "sha1_v1",
34
+ "trace_properties",
35
+ "uuid7",
36
+ "with_request_context",
37
+ ]
numbunch/client.py ADDED
@@ -0,0 +1,520 @@
1
+ """The Numbunch client: batched capture, identity, local flag evaluation and exceptions.
2
+
3
+ The wire format is PostHog's (``/batch/``, ``/flags/?v=2``), so the server needs no new ingestion path. Local
4
+ evaluation reads ``GET /v1/flags/definitions`` with the environment's ``nb_sec_`` secret and evaluates both
5
+ bucketing algorithms; flags it cannot decide (experience continuity, properties the caller did not supply) are asked
6
+ of the server. Parts are shaped after posthog-python (MIT, Copyright (c) PostHog Inc.).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import atexit
12
+ import json
13
+ import logging
14
+ import random
15
+ import sys
16
+ import threading
17
+ import time
18
+ import uuid as uuid_module
19
+ import weakref
20
+ from collections import OrderedDict
21
+ from datetime import datetime, timezone
22
+ from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
23
+
24
+ from . import context
25
+ from .consumer import Consumer
26
+ from .evaluate import NO_GROUP_TYPE, Evaluation, evaluate, payload
27
+ from .exceptions import exception_properties
28
+ from .matching import Inconclusive
29
+ from .transport import Response, Transport, json_body
30
+ from .uuid7 import uuid7
31
+ from .version import VERSION
32
+
33
+ DEFAULT_HOST = "https://in.numbunch.com"
34
+ LIB = "numbunch-python"
35
+
36
+ log = logging.getLogger("numbunch")
37
+
38
+ FlagValue = Union[str, bool]
39
+
40
+
41
+ class _NeedsServer(Exception):
42
+ """A flag the client cannot decide locally."""
43
+
44
+
45
+ def _iso(timestamp: Union[datetime, str, None]) -> str:
46
+ if timestamp is None:
47
+ return datetime.now(timezone.utc).isoformat()
48
+ if isinstance(timestamp, str):
49
+ return timestamp
50
+ if timestamp.tzinfo is None:
51
+ timestamp = timestamp.replace(tzinfo=timezone.utc)
52
+ return timestamp.isoformat()
53
+
54
+
55
+ def _decode_payload(value: Any) -> Any:
56
+ if isinstance(value, str):
57
+ try:
58
+ return json.loads(value)
59
+ except ValueError:
60
+ return value
61
+ return value
62
+
63
+
64
+ class _Definitions:
65
+ """Flag definitions for local evaluation, refreshed with ``If-None-Match`` every ``poll_interval`` seconds."""
66
+
67
+ def __init__(self, client: Numbunch, poll_interval: float) -> None:
68
+ self._client = client
69
+ self._poll_interval = poll_interval
70
+ self._lock = threading.Lock()
71
+ self._etag: Optional[str] = None
72
+ self.flags: Dict[str, Dict[str, Any]] = {}
73
+ self.group_types: Dict[str, str] = {}
74
+ self.loaded = False
75
+ self._stop = threading.Event()
76
+ self._thread: Optional[threading.Thread] = None
77
+
78
+ def start(self) -> None:
79
+ self._thread = threading.Thread(target=self._run, name="numbunch-flags", daemon=True)
80
+ self._thread.start()
81
+
82
+ def stop(self) -> None:
83
+ self._stop.set()
84
+
85
+ def _run(self) -> None:
86
+ while not self._stop.is_set():
87
+ self.load()
88
+ self._stop.wait(self._poll_interval)
89
+
90
+ def ensure_loaded(self) -> bool:
91
+ if not self.loaded:
92
+ self.load()
93
+ return self.loaded
94
+
95
+ def load(self) -> None:
96
+ with self._lock:
97
+ headers = {"Authorization": f"Bearer {self._client.secret_key}"}
98
+ if self._etag:
99
+ headers["If-None-Match"] = self._etag
100
+ try:
101
+ reply = self._client._request("GET", "/v1/flags/definitions", headers=headers)
102
+ except OSError as error:
103
+ self._client._error(f"loading flag definitions failed: {error}")
104
+ return
105
+ if reply.status == 304:
106
+ self.loaded = True
107
+ return
108
+ if reply.status != 200:
109
+ self._client._error(f"loading flag definitions failed: HTTP {reply.status} {reply.body[:200]!r}")
110
+ return
111
+ try:
112
+ body = reply.json() or {}
113
+ except ValueError as error:
114
+ self._client._error(f"flag definitions are not JSON: {error}")
115
+ return
116
+ self.flags = {flag["key"]: flag for flag in body.get("flags") or []}
117
+ self.group_types = {str(k): v for k, v in (body.get("group_type_mapping") or {}).items()}
118
+ self._etag = reply.headers.get("etag")
119
+ self.loaded = True
120
+
121
+
122
+ def _shutdown_at_exit(ref: weakref.ReferenceType[Numbunch]) -> None:
123
+ client = ref()
124
+ if client is not None:
125
+ client.shutdown()
126
+
127
+
128
+ class Numbunch:
129
+ """A Numbunch client. Thread-safe; create one per process and call :meth:`shutdown` (also run at exit).
130
+
131
+ ``api_key`` is the environment's public ``nb_pub_`` token. ``secret_key`` (the environment's ``nb_sec_`` secret)
132
+ turns on local flag evaluation.
133
+ """
134
+
135
+ def __init__(
136
+ self,
137
+ api_key: str,
138
+ host: str = DEFAULT_HOST,
139
+ secret_key: Optional[str] = None,
140
+ flush_at: int = 100,
141
+ flush_interval: float = 10.0,
142
+ max_queue_size: int = 10000,
143
+ poll_interval: float = 30,
144
+ timeout: float = 10,
145
+ max_retries: int = 3,
146
+ disabled: bool = False,
147
+ on_error: Optional[Callable[[str], None]] = None,
148
+ gzip: bool = True,
149
+ transport: Optional[Transport] = None,
150
+ ) -> None:
151
+ self.api_key = api_key
152
+ self.host = host.rstrip("/")
153
+ self.secret_key = secret_key
154
+ self.timeout = timeout
155
+ self.max_retries = max(0, max_retries)
156
+ self.disabled = disabled
157
+ self.gzip = gzip
158
+ self._on_error = on_error
159
+ self._transport = transport or Transport()
160
+ self._flag_calls: OrderedDict[Tuple[str, str, str], None] = OrderedDict()
161
+ self._flag_calls_lock = threading.Lock()
162
+ self._shutdown = False
163
+ self._previous_hooks: Optional[Tuple[Any, Any]] = None
164
+ self._consumer = None if disabled else Consumer(self._send_batch, flush_at, flush_interval, max_queue_size)
165
+ self._definitions: Optional[_Definitions] = None
166
+ if secret_key and not disabled:
167
+ self._definitions = _Definitions(self, poll_interval)
168
+ self._definitions.start()
169
+ atexit.register(_shutdown_at_exit, weakref.ref(self))
170
+
171
+ # ----------------------------------------------------------------------------------------------------- transport
172
+
173
+ def _error(self, message: str) -> None:
174
+ log.warning("numbunch: %s", message)
175
+ if self._on_error is not None:
176
+ try:
177
+ self._on_error(message)
178
+ except Exception: # noqa: BLE001 - a failing callback must not break capture
179
+ log.exception("numbunch: on_error raised")
180
+
181
+ def _request(
182
+ self, method: str, path: str, payload: Any = None, headers: Optional[Dict[str, str]] = None
183
+ ) -> Response:
184
+ body = None
185
+ all_headers = dict(headers or {})
186
+ if payload is not None:
187
+ body, json_headers = json_body(payload, self.gzip)
188
+ all_headers.update(json_headers)
189
+ return self._transport.request(method, f"{self.host}{path}", all_headers, body, self.timeout)
190
+
191
+ def _send_batch(self, batch: List[Dict[str, Any]]) -> None:
192
+ payload = {"api_key": self.api_key, "batch": batch, "sent_at": datetime.now(timezone.utc).isoformat()}
193
+ for attempt in range(self.max_retries + 1):
194
+ try:
195
+ reply = self._request("POST", "/batch/", payload)
196
+ if 200 <= reply.status < 300:
197
+ return
198
+ retryable = reply.status >= 500 or reply.status == 429
199
+ problem = f"HTTP {reply.status} {reply.body[:200]!r}"
200
+ except (ValueError, TypeError, RecursionError) as error:
201
+ # Isolate a malformed property without losing valid events in the same batch.
202
+ if len(batch) == 1:
203
+ self._error(f"invalid event payload: {type(error).__name__}")
204
+ else:
205
+ middle = len(batch) // 2
206
+ self._send_batch(batch[:middle])
207
+ self._send_batch(batch[middle:])
208
+ return
209
+ except OSError as error:
210
+ retryable, problem = True, str(error)
211
+ if not retryable or attempt == self.max_retries:
212
+ self._error(f"sending {len(batch)} events failed: {problem}")
213
+ return
214
+ # Exponential backoff with jitter: 0.5 s, 1 s, 2 s, … (at most 30 s), times 0.5–1.5.
215
+ time.sleep(min(30.0, 0.5 * 2**attempt) * (0.5 + random.random()))
216
+
217
+ # ------------------------------------------------------------------------------------------------------ capture
218
+
219
+ def capture(
220
+ self,
221
+ distinct_id: Optional[str],
222
+ event: str,
223
+ properties: Optional[Mapping[str, Any]] = None,
224
+ groups: Optional[Mapping[str, str]] = None,
225
+ uuid: Union[str, uuid_module.UUID, None] = None,
226
+ timestamp: Union[datetime, str, None] = None,
227
+ ) -> Optional[str]:
228
+ """Queues an event and returns its UUID (``None`` if disabled or the queue is full).
229
+
230
+ Pass ``uuid`` and ``timestamp`` for idempotent server events: the server deduplicates on the UUID. Without a
231
+ ``distinct_id`` the request context's ``user.id`` is used, else the event is personless.
232
+ """
233
+ if self.disabled or self._consumer is None:
234
+ return None
235
+ props: Dict[str, Any] = dict(context.trace_properties())
236
+ session_id = context.current_session_id()
237
+ if session_id:
238
+ props["$session_id"] = session_id
239
+ props.update(properties or {})
240
+ if distinct_id is None:
241
+ distinct_id = context.current_distinct_id()
242
+ if distinct_id is None:
243
+ distinct_id = str(uuid7())
244
+ props.setdefault("$process_person_profile", False)
245
+ if groups:
246
+ props["$groups"] = dict(groups)
247
+ props["$lib"] = LIB
248
+ props["$lib_version"] = VERSION
249
+ event_uuid = str(uuid or uuid7())
250
+ message = {
251
+ "type": "capture",
252
+ "event": event,
253
+ "distinct_id": str(distinct_id),
254
+ "properties": props,
255
+ "timestamp": _iso(timestamp),
256
+ "uuid": event_uuid,
257
+ }
258
+ if not self._consumer.put(message):
259
+ self._error(f"queue full or shut down; dropped {event!r}")
260
+ return None
261
+ return event_uuid
262
+
263
+ def identify(
264
+ self, distinct_id: str, properties: Optional[Mapping[str, Any]] = None, **kwargs: Any
265
+ ) -> Optional[str]:
266
+ """Sets person properties (``$set``) on ``distinct_id``'s person."""
267
+ return self.capture(distinct_id, "$identify", {"$set": dict(properties or {})}, **kwargs)
268
+
269
+ def group_identify(
270
+ self,
271
+ group_type: str,
272
+ group_key: str,
273
+ properties: Optional[Mapping[str, Any]] = None,
274
+ distinct_id: Optional[str] = None,
275
+ **kwargs: Any,
276
+ ) -> Optional[str]:
277
+ """Sets properties of a group (``$groupidentify``)."""
278
+ props = {"$group_type": group_type, "$group_key": group_key, "$group_set": dict(properties or {})}
279
+ return self.capture(distinct_id or f"${group_type}_{group_key}", "$groupidentify", props, **kwargs)
280
+
281
+ def capture_exception(
282
+ self,
283
+ exc: Optional[BaseException] = None,
284
+ distinct_id: Optional[str] = None,
285
+ properties: Optional[Mapping[str, Any]] = None,
286
+ handled: bool = True,
287
+ ) -> Optional[str]:
288
+ """Captures ``exc`` (default: the exception being handled) as ``$exception``."""
289
+ props = exception_properties(exc, handled)
290
+ if props is None:
291
+ return None
292
+ props.update(properties or {})
293
+ return self.capture(distinct_id, "$exception", props)
294
+
295
+ def enable_exception_autocapture(self) -> None:
296
+ """Captures uncaught exceptions (``sys.excepthook`` and ``threading.excepthook``), then calls the previous
297
+ hooks."""
298
+ if self._previous_hooks is not None:
299
+ return
300
+ previous_sys, previous_thread = sys.excepthook, threading.excepthook
301
+ self._previous_hooks = (previous_sys, previous_thread)
302
+
303
+ def sys_hook(exc_type: Any, exc: BaseException, tb: Any) -> None:
304
+ if not isinstance(exc, KeyboardInterrupt):
305
+ self.capture_exception(exc, handled=False)
306
+ self.flush(timeout=self.timeout)
307
+ previous_sys(exc_type, exc, tb)
308
+
309
+ def thread_hook(args: Any) -> None:
310
+ if args.exc_value is not None and not isinstance(args.exc_value, SystemExit):
311
+ self.capture_exception(args.exc_value, handled=False)
312
+ previous_thread(args)
313
+
314
+ sys.excepthook = sys_hook
315
+ threading.excepthook = thread_hook
316
+
317
+ # -------------------------------------------------------------------------------------------------------- flags
318
+
319
+ def _local(
320
+ self,
321
+ flag: Mapping[str, Any],
322
+ distinct_id: str,
323
+ person_properties: Optional[Mapping[str, Any]],
324
+ groups: Optional[Mapping[str, str]],
325
+ group_properties: Optional[Mapping[str, Mapping[str, Any]]],
326
+ ) -> Evaluation:
327
+ if flag.get("ensure_continuity"):
328
+ raise _NeedsServer
329
+ filters = flag.get("filters") or {}
330
+ index = filters.get("aggregation_group_type_index")
331
+ try:
332
+ if index is not None:
333
+ group_types = self._definitions.group_types if self._definitions is not None else {}
334
+ name = group_types.get(str(index))
335
+ key = (groups or {}).get(name) if name else None
336
+ if key is None or name is None:
337
+ if not flag.get("active", True):
338
+ return evaluate(flag, "", {})
339
+ return Evaluation(False, None, NO_GROUP_TYPE)
340
+ props: Dict[str, Any] = {"$group_key": key, **dict((group_properties or {}).get(name) or {})}
341
+ result = evaluate(flag, str(key), props)
342
+ else:
343
+ props = dict(person_properties or {})
344
+ props.setdefault("distinct_id", distinct_id)
345
+ result = evaluate(flag, distinct_id, props)
346
+ except Inconclusive as error:
347
+ raise _NeedsServer from error
348
+ if result.skipped_inconclusive:
349
+ raise _NeedsServer
350
+ return result
351
+
352
+ def _remote(
353
+ self,
354
+ distinct_id: str,
355
+ person_properties: Optional[Mapping[str, Any]],
356
+ groups: Optional[Mapping[str, str]],
357
+ group_properties: Optional[Mapping[str, Mapping[str, Any]]],
358
+ keys: Optional[List[str]] = None,
359
+ ) -> Optional[Dict[str, Dict[str, Any]]]:
360
+ body: Dict[str, Any] = {
361
+ "token": self.api_key,
362
+ "distinct_id": distinct_id,
363
+ "person_properties": dict(person_properties or {}),
364
+ "groups": dict(groups or {}),
365
+ "group_properties": {k: dict(v) for k, v in (group_properties or {}).items()},
366
+ }
367
+ if keys is not None:
368
+ body["flag_keys_to_evaluate"] = keys
369
+ try:
370
+ reply = self._request("POST", "/flags/?v=2", body)
371
+ except OSError as error:
372
+ self._error(f"remote flag evaluation failed: {error}")
373
+ return None
374
+ if reply.status != 200:
375
+ self._error(f"remote flag evaluation failed: HTTP {reply.status}")
376
+ return None
377
+ try:
378
+ flags = (reply.json() or {}).get("flags") or {}
379
+ except ValueError:
380
+ return None
381
+ out = {}
382
+ for key, detail in flags.items():
383
+ variant = detail.get("variant")
384
+ value = variant if variant is not None else bool(detail.get("enabled"))
385
+ out[key] = {"value": value, "payload": (detail.get("metadata") or {}).get("payload")}
386
+ return out
387
+
388
+ def _detail(
389
+ self,
390
+ key: str,
391
+ distinct_id: str,
392
+ person_properties: Optional[Mapping[str, Any]],
393
+ groups: Optional[Mapping[str, str]],
394
+ group_properties: Optional[Mapping[str, Mapping[str, Any]]],
395
+ only_evaluate_locally: bool,
396
+ ) -> Optional[Tuple[FlagValue, Any, bool]]:
397
+ """``(value, payload, locally_evaluated)``, or ``None`` when the flag is unknown or undecidable."""
398
+ if self.disabled:
399
+ return None
400
+ if self._definitions is not None and self._definitions.ensure_loaded():
401
+ flag = self._definitions.flags.get(key)
402
+ if flag is not None:
403
+ try:
404
+ result = self._local(flag, distinct_id, person_properties, groups, group_properties)
405
+ return result.value, payload(flag, result), True
406
+ except _NeedsServer:
407
+ pass
408
+ if only_evaluate_locally:
409
+ return None
410
+ remote = self._remote(distinct_id, person_properties, groups, group_properties, [key])
411
+ if remote is None or key not in remote:
412
+ return None
413
+ return remote[key]["value"], remote[key]["payload"], False
414
+
415
+ def get_feature_flag(
416
+ self,
417
+ key: str,
418
+ distinct_id: str,
419
+ person_properties: Optional[Mapping[str, Any]] = None,
420
+ groups: Optional[Mapping[str, str]] = None,
421
+ group_properties: Optional[Mapping[str, Mapping[str, Any]]] = None,
422
+ only_evaluate_locally: bool = False,
423
+ send_feature_flag_events: bool = True,
424
+ ) -> Optional[FlagValue]:
425
+ """The flag's variant, ``True``/``False``, or ``None`` if it is unknown or could not be evaluated."""
426
+ detail = self._detail(key, distinct_id, person_properties, groups, group_properties, only_evaluate_locally)
427
+ if detail is None:
428
+ return None
429
+ if send_feature_flag_events:
430
+ self._flag_called(key, distinct_id, detail[0], detail[2], groups)
431
+ return detail[0]
432
+
433
+ def feature_enabled(self, key: str, distinct_id: str, **kwargs: Any) -> Optional[bool]:
434
+ value = self.get_feature_flag(key, distinct_id, **kwargs)
435
+ return None if value is None else bool(value)
436
+
437
+ def get_feature_flag_payload(
438
+ self,
439
+ key: str,
440
+ distinct_id: str,
441
+ person_properties: Optional[Mapping[str, Any]] = None,
442
+ groups: Optional[Mapping[str, str]] = None,
443
+ group_properties: Optional[Mapping[str, Mapping[str, Any]]] = None,
444
+ only_evaluate_locally: bool = False,
445
+ ) -> Any:
446
+ """The payload for the flag's current value (JSON-decoded), or ``None``."""
447
+ detail = self._detail(key, distinct_id, person_properties, groups, group_properties, only_evaluate_locally)
448
+ return None if detail is None else _decode_payload(detail[1])
449
+
450
+ def get_all_flags(
451
+ self,
452
+ distinct_id: str,
453
+ person_properties: Optional[Mapping[str, Any]] = None,
454
+ groups: Optional[Mapping[str, str]] = None,
455
+ group_properties: Optional[Mapping[str, Mapping[str, Any]]] = None,
456
+ only_evaluate_locally: bool = False,
457
+ ) -> Dict[str, FlagValue]:
458
+ """Every flag's value for ``distinct_id``: local where possible, one server request for the rest."""
459
+ if self.disabled:
460
+ return {}
461
+ out: Dict[str, FlagValue] = {}
462
+ needs_server = True
463
+ if self._definitions is not None and self._definitions.ensure_loaded():
464
+ needs_server = False
465
+ for key, flag in self._definitions.flags.items():
466
+ try:
467
+ out[key] = self._local(flag, distinct_id, person_properties, groups, group_properties).value
468
+ except _NeedsServer:
469
+ needs_server = True
470
+ if needs_server and not only_evaluate_locally:
471
+ remote = self._remote(distinct_id, person_properties, groups, group_properties)
472
+ for key, detail in (remote or {}).items():
473
+ out.setdefault(key, detail["value"])
474
+ return out
475
+
476
+ def get_bootstrap(self, distinct_id: str, **kwargs: Any) -> Dict[str, Any]:
477
+ """``{"distinctId", "flags"}`` to embed in HTML as ``@numbunch/browser``'s ``bootstrap`` option, so the page
478
+ reads flags before any request."""
479
+ return {"distinctId": distinct_id, "flags": self.get_all_flags(distinct_id, **kwargs)}
480
+
481
+ def _flag_called(
482
+ self, key: str, distinct_id: str, value: FlagValue, locally: bool, groups: Optional[Mapping[str, str]]
483
+ ) -> None:
484
+ cache_key = (distinct_id, key, json.dumps(value))
485
+ with self._flag_calls_lock:
486
+ if cache_key in self._flag_calls:
487
+ self._flag_calls.move_to_end(cache_key)
488
+ return
489
+ self._flag_calls[cache_key] = None
490
+ if len(self._flag_calls) > 50_000:
491
+ self._flag_calls.popitem(last=False)
492
+ properties = {
493
+ "$feature_flag": key,
494
+ "$feature_flag_response": value,
495
+ "locally_evaluated": locally,
496
+ f"$feature/{key}": value,
497
+ }
498
+ self.capture(distinct_id, "$feature_flag_called", properties, groups=groups)
499
+
500
+ # ----------------------------------------------------------------------------------------------------- lifecycle
501
+
502
+ def flush(self, timeout: Optional[float] = None) -> bool:
503
+ """Blocks until every queued event is sent (or ``timeout`` seconds pass)."""
504
+ return True if self._consumer is None else self._consumer.flush(timeout)
505
+
506
+ def shutdown(self, timeout: Optional[float] = None) -> None:
507
+ """Flushes, stops polling and stops the background threads. Idempotent."""
508
+ if self._shutdown:
509
+ return
510
+ self._shutdown = True
511
+ if self._definitions is not None:
512
+ self._definitions.stop()
513
+ if self._consumer is not None:
514
+ self._consumer.stop(timeout)
515
+
516
+ def __enter__(self) -> Numbunch:
517
+ return self
518
+
519
+ def __exit__(self, *exc: Any) -> None:
520
+ self.shutdown()
numbunch/consumer.py ADDED
@@ -0,0 +1,95 @@
1
+ """The background batch queue: events are queued by ``capture`` and sent by one daemon thread in batches of up to
2
+ ``flush_at`` events, at least every ``flush_interval`` seconds."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import queue
7
+ import threading
8
+ import time
9
+ from typing import Any, Callable, Dict, List, Optional
10
+
11
+
12
+ class Consumer:
13
+ def __init__(
14
+ self,
15
+ send: Callable[[List[Dict[str, Any]]], None],
16
+ flush_at: int,
17
+ flush_interval: float,
18
+ max_queue_size: int,
19
+ ) -> None:
20
+ self._send = send
21
+ self._flush_at = max(1, flush_at)
22
+ self._flush_interval = max(0.0, flush_interval)
23
+ self._queue: queue.Queue[Dict[str, Any]] = queue.Queue(max_queue_size)
24
+ self._flush_now = threading.Event()
25
+ self._stopping = threading.Event()
26
+ self._thread = threading.Thread(target=self._run, name="numbunch-consumer", daemon=True)
27
+ self._thread.start()
28
+
29
+ def put(self, event: Dict[str, Any]) -> bool:
30
+ """Queues an event; ``False`` when the queue is full (the event is dropped)."""
31
+ if self._stopping.is_set():
32
+ return False
33
+ try:
34
+ self._queue.put_nowait(event)
35
+ except queue.Full:
36
+ return False
37
+ if self._queue.qsize() >= self._flush_at:
38
+ self._flush_now.set()
39
+ return True
40
+
41
+ def flush(self, timeout: Optional[float] = None) -> bool:
42
+ """Sends everything queued; ``False`` if ``timeout`` (seconds) passed first."""
43
+ deadline = None if timeout is None else time.monotonic() + timeout
44
+ done = self._queue.all_tasks_done
45
+ with done:
46
+ while self._queue.unfinished_tasks:
47
+ if not self._thread.is_alive():
48
+ return False
49
+ self._flush_now.set()
50
+ wait = 0.05 if deadline is None else min(0.05, deadline - time.monotonic())
51
+ if wait <= 0:
52
+ return False
53
+ done.wait(wait)
54
+ return True
55
+
56
+ def stop(self, timeout: Optional[float] = None) -> None:
57
+ """Sends what is queued and stops the thread."""
58
+ self.flush(timeout)
59
+ self._stopping.set()
60
+ self._thread.join(timeout)
61
+
62
+ def _run(self) -> None:
63
+ while True:
64
+ first = self._first()
65
+ if first is None:
66
+ return
67
+ batch = [first]
68
+ deadline = time.monotonic() + self._flush_interval
69
+ while len(batch) < self._flush_at:
70
+ urgent = self._flush_now.is_set() or self._stopping.is_set()
71
+ remaining = deadline - time.monotonic()
72
+ try:
73
+ if urgent or remaining <= 0:
74
+ batch.append(self._queue.get_nowait())
75
+ else:
76
+ batch.append(self._queue.get(timeout=min(remaining, 0.05)))
77
+ except queue.Empty:
78
+ if urgent or remaining <= 0:
79
+ break
80
+ try:
81
+ self._send(batch)
82
+ finally:
83
+ for _ in batch:
84
+ self._queue.task_done()
85
+ if self._queue.empty():
86
+ self._flush_now.clear()
87
+
88
+ def _first(self) -> Optional[Dict[str, Any]]:
89
+ """Blocks until an event arrives; ``None`` once stopping with nothing left."""
90
+ while True:
91
+ try:
92
+ return self._queue.get(timeout=0.1)
93
+ except queue.Empty:
94
+ if self._stopping.is_set():
95
+ return None