ringo-task-queue 0.1.0.dev6__py3-none-win_amd64.whl → 0.1.0.dev7__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,480 +1,480 @@
1
- """Process-local dynamic Worker concurrency controllers.
2
-
3
- Controllers are deliberately small asyncio-local admission pools. A Worker
4
- holds at most one speculative (probe) credit; consuming it turns the credit
5
- into a running permit and immediately lets the controller grant another
6
- member. The controller never cancels a running handler.
7
- """
8
-
9
- from __future__ import annotations
10
-
11
- import asyncio
12
- from dataclasses import dataclass
13
- from typing import TYPE_CHECKING
14
-
15
- from .errors import UnavailableError
16
-
17
- if TYPE_CHECKING: # pragma: no cover
18
- from .worker import Worker
19
-
20
- _MAX_CAPACITY = 2**31 - 1
21
- _PROBE_SECONDS = 0.25
22
-
23
-
24
- @dataclass
25
- class _Member:
26
- worker: "Worker"
27
- weight: int
28
- deficit: int = 0
29
- probe: bool = False
30
- revoking: bool = False
31
- active: bool = False
32
- cooldown_until: float = 0.0
33
- running: int = 0
34
- attached: bool = True
35
- session_ready: bool = False
36
- probe_generation: int = 0
37
-
38
-
39
- class ResourceConcurrency:
40
- """A shareable, process-local resource capacity pool.
41
-
42
- ``capacity`` is the total number of admission permits, not a per-worker
43
- quota. The same object may be supplied to workers on different queues or
44
- clients in one event loop. Capacity changes are serialized and are
45
- non-preemptive: existing handlers continue to hold their permits.
46
- """
47
-
48
- def __init__(self, capacity: int) -> None:
49
- self._validate_capacity(capacity)
50
- self._capacity = capacity
51
- self._members: list[_Member] = []
52
- self._lock = asyncio.Lock()
53
- # Capacity transitions stay linearizable across the network wait used
54
- # to withdraw outstanding probe credits.
55
- self._resize_lock = asyncio.Lock()
56
- self._cursor = 0
57
- self._closed = False
58
- self._grant_paused = False
59
- self._resize_needs_retry = False
60
- self._loop: asyncio.AbstractEventLoop | None = None
61
-
62
- @staticmethod
63
- def _validate_capacity(value: int) -> None:
64
- if isinstance(value, bool) or not isinstance(value, int):
65
- raise ValueError("capacity must be an integer")
66
- if value < 0 or value > _MAX_CAPACITY:
67
- raise ValueError(f"capacity must be between 0 and {_MAX_CAPACITY}")
68
-
69
- @property
70
- def capacity(self) -> int:
71
- return self._capacity
72
-
73
- @property
74
- def members(self) -> int:
75
- """Number of currently attached workers (primarily for diagnostics)."""
76
- return sum(1 for member in self._members if member.attached)
77
-
78
- async def set_capacity(self, capacity: int) -> None:
79
- """Publish a new capacity and await withdrawal of idle probes."""
80
- self._validate_capacity(capacity)
81
- self._check_loop()
82
- async with self._resize_lock:
83
- async with self._lock:
84
- probes = self._prepare_resize_locked(capacity)
85
- await self._finish_resize(capacity, probes)
86
-
87
- def _prepare_resize_locked(self, capacity: int) -> list[_Member]:
88
- if self._closed:
89
- raise RuntimeError("concurrency controller is closed")
90
- old = self._capacity
91
- self._capacity = capacity
92
- probes: list[_Member] = []
93
- if capacity != old or self._resize_needs_retry:
94
- # Pause every grant while all live streams receive the new target.
95
- # Existing probes are withdrawn and then re-granted under that
96
- # target; assignment-before-result still promotes the reservation.
97
- self._grant_paused = True
98
- probes = [
99
- member
100
- for member in self._members
101
- if member.attached and member.probe
102
- ]
103
- for member in probes:
104
- if not member.revoking:
105
- member.revoking = True
106
- member.probe_generation += 1
107
- return probes
108
-
109
- async def _finish_resize(self, capacity: int, probes: list[_Member]) -> None:
110
- async with self._lock:
111
- if not self._grant_paused:
112
- await self._grant_locked()
113
- return
114
- probe_ids = {id(member) for member in probes}
115
- actions = [
116
- (member, id(member) in probe_ids)
117
- for member in self._members
118
- if member.attached and member.session_ready
119
- ]
120
- results = await asyncio.gather(
121
- *(
122
- member.worker._withdraw_controller_probe(member, capacity)
123
- if has_probe
124
- else member.worker._publish_controller_target(capacity)
125
- for member, has_probe in actions
126
- ),
127
- return_exceptions=True,
128
- )
129
- failed = False
130
- async with self._lock:
131
- for (member, has_probe), result in zip(actions, results):
132
- if result is not True:
133
- failed = True
134
- elif has_probe and member.probe and member.revoking:
135
- member.probe = False
136
- member.revoking = False
137
- self._resize_needs_retry = failed
138
- self._grant_paused = failed
139
- if not failed:
140
- await self._grant_locked()
141
- if failed:
142
- raise UnavailableError(
143
- "concurrency capacity changed locally, but an old Work probe "
144
- "or target snapshot could not be published; retry after the "
145
- "stream reconnects"
146
- )
147
-
148
- async def close(self) -> None:
149
- """Stop future grants and withdraw attached idle probes."""
150
- self._check_loop()
151
- async with self._resize_lock:
152
- async with self._lock:
153
- self._closed = True
154
- self._grant_paused = True
155
- probes = [member for member in self._members if member.probe]
156
- for member in probes:
157
- member.revoking = True
158
- member.probe_generation += 1
159
- results: list[object] = []
160
- if probes:
161
- results = await asyncio.gather(
162
- *(
163
- member.worker._withdraw_controller_probe(
164
- member, 0, force_target=True
165
- )
166
- for member in probes
167
- ),
168
- return_exceptions=True,
169
- )
170
- async with self._lock:
171
- for member, result in zip(probes, results):
172
- if result is True and member.probe and member.revoking:
173
- member.probe = False
174
- member.revoking = False
175
-
176
- # The following methods are Worker integration points. They are kept
177
- # private so a controller cannot be confused with a transport-level quota.
178
- def _check_loop(self) -> None:
179
- loop = asyncio.get_running_loop()
180
- if self._loop is None:
181
- self._loop = loop
182
- elif self._loop is not loop:
183
- raise RuntimeError("a concurrency controller cannot cross event loops")
184
-
185
- def _attach(self, worker: "Worker", weight: int) -> _Member:
186
- if self._closed:
187
- raise RuntimeError("concurrency controller is closed")
188
- if isinstance(weight, bool) or not isinstance(weight, int) or weight < 1:
189
- raise ValueError("weight must be a positive integer")
190
- for member in self._members:
191
- if member.worker is worker:
192
- return member
193
- member = _Member(worker=worker, weight=weight)
194
- self._members.append(member)
195
- return member
196
-
197
- async def _start_member(self, member: _Member | None) -> None:
198
- if member is None:
199
- return
200
- self._check_loop()
201
- # Session admission and capacity transitions share the complete resize
202
- # lane, including the initial WorkResult wait. A stale initial target
203
- # therefore cannot land after a completed resize.
204
- async with self._resize_lock:
205
- await member.worker._publish_controller_initial(self._capacity)
206
- async with self._lock:
207
- if member.attached and not self._closed:
208
- member.session_ready = True
209
- await self._grant_locked()
210
-
211
- async def _detach(self, member: _Member) -> None:
212
- self._check_loop()
213
- async with self._resize_lock:
214
- async with self._lock:
215
- if not member.attached:
216
- return
217
- member.attached = False
218
- member.session_ready = False
219
- needs_withdraw = member.probe and not member.revoking
220
- if needs_withdraw:
221
- member.revoking = True
222
- member.probe_generation += 1
223
- withdrawn = True
224
- if needs_withdraw:
225
- withdrawn = await member.worker._withdraw_controller_probe(
226
- member, 0, force_target=True
227
- )
228
- async with self._lock:
229
- if withdrawn is True and member.probe and member.revoking:
230
- member.probe = False
231
- member.revoking = False
232
- await self._grant_locked()
233
- self._prune_detached()
234
-
235
- def _prune_detached(self) -> None:
236
- self._members[:] = [
237
- member
238
- for member in self._members
239
- if member.attached or member.running or member.probe
240
- ]
241
- if self._members:
242
- self._cursor %= len(self._members)
243
- else:
244
- self._cursor = 0
245
-
246
- async def _stream_lost(self, member: _Member | None) -> None:
247
- """Return a probe whose old Work stream disappeared.
248
-
249
- The server-side credit vanished with that stream. Running permits are
250
- deliberately untouched; only the unused probe is made available for
251
- republication after the replacement session registers.
252
- """
253
- if member is None:
254
- return
255
- self._check_loop()
256
- async with self._lock:
257
- member.session_ready = False
258
- if member.probe:
259
- member.probe = False
260
- member.revoking = False
261
- member.probe_generation += 1
262
- await self._grant_locked()
263
-
264
- async def _probe_consumed(self, member: _Member) -> bool:
265
- self._check_loop()
266
- async with self._lock:
267
- if not member.probe:
268
- return False
269
- # This also handles assignment-before-revoke-result: the probe is
270
- # promoted to running and the in-flight withdrawal must not release
271
- # the resulting permit.
272
- member.probe = False
273
- member.revoking = False
274
- member.active = True
275
- member.running += 1
276
- await self._grant_locked()
277
- return True
278
-
279
- async def _release_running(
280
- self,
281
- member: _Member,
282
- *,
283
- success: bool = False,
284
- congestion: bool = False,
285
- ) -> None:
286
- self._check_loop()
287
- # Automatic AIMD transitions and manual set_capacity calls share one
288
- # end-to-end resize lane, including the correlated withdrawal wait.
289
- async with self._resize_lock:
290
- resize_capacity: int | None = None
291
- probes: list[_Member] = []
292
- async with self._lock:
293
- controller = self
294
- saturated = self._admission_count() >= self._capacity
295
- if member.running:
296
- member.running -= 1
297
- if self._closed:
298
- self._prune_detached()
299
- return
300
- if isinstance(controller, AdaptiveConcurrency):
301
- if success and saturated and self._capacity > 0:
302
- controller._successes += 1
303
- controller._successes = min(
304
- controller._successes, self._capacity
305
- )
306
- if controller._successes >= self._capacity:
307
- controller._successes = 0
308
- new_capacity = min(
309
- _MAX_CAPACITY,
310
- controller._capacity + 1,
311
- controller.maximum,
312
- )
313
- if new_capacity != controller._capacity:
314
- resize_capacity = new_capacity
315
- probes = self._prepare_resize_locked(new_capacity)
316
- elif congestion:
317
- controller._successes = 0
318
- resize_capacity = max(
319
- controller.minimum,
320
- int(controller._capacity * controller.mult_decrease),
321
- )
322
- if resize_capacity < controller._capacity:
323
- # Stop grants and reserve every probe before releasing
324
- # the lock; withdrawal itself happens outside it.
325
- probes = self._prepare_resize_locked(resize_capacity)
326
- if resize_capacity is None:
327
- await self._grant_locked()
328
- self._prune_detached()
329
- if resize_capacity is not None:
330
- await self._finish_resize(resize_capacity, probes)
331
-
332
- def _admission_count(self) -> int:
333
- return sum(member.running + int(member.probe) for member in self._members)
334
-
335
- def _choose_member(self) -> _Member | None:
336
- """Choose with smooth weighted round-robin admission.
337
-
338
- Members whose probes have expired enter a short cooldown before the
339
- same weighted rotation can issue a bounded discovery probe again. A
340
- continuously consuming member therefore remains work-conserving while
341
- an idle member is never permanently starved.
342
- """
343
- now = asyncio.get_running_loop().time()
344
- eligible = [
345
- member
346
- for member in self._members
347
- if member.attached
348
- and member.session_ready
349
- and not member.probe
350
- and (member.active or now >= member.cooldown_until)
351
- ]
352
- if not eligible:
353
- return None
354
- total = sum(member.weight for member in eligible)
355
- count = len(self._members)
356
- ordered = [
357
- self._members[(self._cursor + i) % count]
358
- for i in range(count)
359
- if self._members[(self._cursor + i) % count] in eligible
360
- ]
361
- for member in eligible:
362
- member.deficit += member.weight
363
- best = max(ordered, key=lambda member: member.deficit)
364
- best.deficit -= total
365
- self._cursor = (self._members.index(best) + 1) % count
366
- return best
367
-
368
- async def _grant_locked(self) -> None:
369
- if self._closed or self._grant_paused:
370
- return
371
- while self._admission_count() < self._capacity:
372
- member = self._choose_member()
373
- if member is None:
374
- return
375
- member.probe = True
376
- member.probe_generation += 1
377
- generation = member.probe_generation
378
- asyncio.create_task(
379
- self._publish_probe(member, generation),
380
- name=f"ringo-concurrency-probe-{member.worker.worker_id}",
381
- )
382
-
383
- async def _publish_probe(self, member: _Member, generation: int) -> None:
384
- try:
385
- await member.worker._publish_controller_probe(
386
- member, self._capacity, generation
387
- )
388
- # An unused probe is a temporary admission reservation only. It
389
- # must be withdrawn even when the server has no task for a member.
390
- asyncio.create_task(
391
- self._expire_probe(member, generation),
392
- name=f"ringo-probe-expiry-{member.worker.worker_id}",
393
- )
394
- except Exception:
395
- async with self._lock:
396
- if member.probe and member.probe_generation == generation:
397
- member.probe = False
398
- member.active = False
399
- member.cooldown_until = (
400
- asyncio.get_running_loop().time() + _PROBE_SECONDS
401
- )
402
- asyncio.create_task(
403
- self._wake_member(member),
404
- name=f"ringo-probe-failure-{member.worker.worker_id}",
405
- )
406
- await self._grant_locked()
407
-
408
- async def _expire_probe(self, member: _Member, generation: int) -> None:
409
- await asyncio.sleep(_PROBE_SECONDS)
410
- async with self._lock:
411
- if (
412
- self._closed
413
- or not member.probe
414
- or member.revoking
415
- or member.probe_generation != generation
416
- ):
417
- return
418
- member.revoking = True
419
- member.active = False
420
- member.probe_generation += 1
421
- capacity = self._capacity
422
- withdrawn = await member.worker._withdraw_controller_probe(member, capacity)
423
- async with self._lock:
424
- if withdrawn is True and member.probe and member.revoking:
425
- member.probe = False
426
- member.revoking = False
427
- member.cooldown_until = (
428
- asyncio.get_running_loop().time() + _PROBE_SECONDS
429
- )
430
- asyncio.create_task(
431
- self._wake_member(member),
432
- name=f"ringo-probe-cooldown-{member.worker.worker_id}",
433
- )
434
- await self._grant_locked()
435
-
436
- async def _wake_member(self, member: _Member) -> None:
437
- delay = max(0.0, member.cooldown_until - asyncio.get_running_loop().time())
438
- if delay:
439
- await asyncio.sleep(delay)
440
- async with self._lock:
441
- if member.attached and not member.probe and not self._closed:
442
- await self._grant_locked()
443
-
444
-
445
- class AdaptiveConcurrency(ResourceConcurrency):
446
- """AIMD resource controller (additive increase, multiplicative decrease)."""
447
-
448
- def __init__(
449
- self,
450
- initial: int,
451
- *,
452
- minimum: int = 1,
453
- maximum: int = _MAX_CAPACITY,
454
- ) -> None:
455
- self._validate_capacity(minimum)
456
- self._validate_capacity(maximum)
457
- self._validate_capacity(initial)
458
- if minimum > maximum:
459
- raise ValueError("minimum must not exceed maximum")
460
- if not minimum <= initial <= maximum:
461
- raise ValueError("initial must be between minimum and maximum")
462
- super().__init__(initial)
463
- self.minimum = minimum
464
- self.maximum = maximum
465
- self.mult_decrease = 0.5
466
- self._successes = 0
467
-
468
- async def set_capacity(self, capacity: int) -> None:
469
- self._validate_capacity(capacity)
470
- if not self.minimum <= capacity <= self.maximum:
471
- raise ValueError("capacity must be between minimum and maximum")
472
- self._check_loop()
473
- async with self._resize_lock:
474
- async with self._lock:
475
- self._successes = 0
476
- probes = self._prepare_resize_locked(capacity)
477
- await self._finish_resize(capacity, probes)
478
-
479
-
480
- __all__ = ["AdaptiveConcurrency", "ResourceConcurrency"]
1
+ """Process-local dynamic Worker concurrency controllers.
2
+
3
+ Controllers are deliberately small asyncio-local admission pools. A Worker
4
+ holds at most one speculative (probe) credit; consuming it turns the credit
5
+ into a running permit and immediately lets the controller grant another
6
+ member. The controller never cancels a running handler.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ from dataclasses import dataclass
13
+ from typing import TYPE_CHECKING
14
+
15
+ from .errors import UnavailableError
16
+
17
+ if TYPE_CHECKING: # pragma: no cover
18
+ from .worker import Worker
19
+
20
+ _MAX_CAPACITY = 2**31 - 1
21
+ _PROBE_SECONDS = 0.25
22
+
23
+
24
+ @dataclass
25
+ class _Member:
26
+ worker: "Worker"
27
+ weight: int
28
+ deficit: int = 0
29
+ probe: bool = False
30
+ revoking: bool = False
31
+ active: bool = False
32
+ cooldown_until: float = 0.0
33
+ running: int = 0
34
+ attached: bool = True
35
+ session_ready: bool = False
36
+ probe_generation: int = 0
37
+
38
+
39
+ class ResourceConcurrency:
40
+ """A shareable, process-local resource capacity pool.
41
+
42
+ ``capacity`` is the total number of admission permits, not a per-worker
43
+ quota. The same object may be supplied to workers on different queues or
44
+ clients in one event loop. Capacity changes are serialized and are
45
+ non-preemptive: existing handlers continue to hold their permits.
46
+ """
47
+
48
+ def __init__(self, capacity: int) -> None:
49
+ self._validate_capacity(capacity)
50
+ self._capacity = capacity
51
+ self._members: list[_Member] = []
52
+ self._lock = asyncio.Lock()
53
+ # Capacity transitions stay linearizable across the network wait used
54
+ # to withdraw outstanding probe credits.
55
+ self._resize_lock = asyncio.Lock()
56
+ self._cursor = 0
57
+ self._closed = False
58
+ self._grant_paused = False
59
+ self._resize_needs_retry = False
60
+ self._loop: asyncio.AbstractEventLoop | None = None
61
+
62
+ @staticmethod
63
+ def _validate_capacity(value: int) -> None:
64
+ if isinstance(value, bool) or not isinstance(value, int):
65
+ raise ValueError("capacity must be an integer")
66
+ if value < 0 or value > _MAX_CAPACITY:
67
+ raise ValueError(f"capacity must be between 0 and {_MAX_CAPACITY}")
68
+
69
+ @property
70
+ def capacity(self) -> int:
71
+ return self._capacity
72
+
73
+ @property
74
+ def members(self) -> int:
75
+ """Number of currently attached workers (primarily for diagnostics)."""
76
+ return sum(1 for member in self._members if member.attached)
77
+
78
+ async def set_capacity(self, capacity: int) -> None:
79
+ """Publish a new capacity and await withdrawal of idle probes."""
80
+ self._validate_capacity(capacity)
81
+ self._check_loop()
82
+ async with self._resize_lock:
83
+ async with self._lock:
84
+ probes = self._prepare_resize_locked(capacity)
85
+ await self._finish_resize(capacity, probes)
86
+
87
+ def _prepare_resize_locked(self, capacity: int) -> list[_Member]:
88
+ if self._closed:
89
+ raise RuntimeError("concurrency controller is closed")
90
+ old = self._capacity
91
+ self._capacity = capacity
92
+ probes: list[_Member] = []
93
+ if capacity != old or self._resize_needs_retry:
94
+ # Pause every grant while all live streams receive the new target.
95
+ # Existing probes are withdrawn and then re-granted under that
96
+ # target; assignment-before-result still promotes the reservation.
97
+ self._grant_paused = True
98
+ probes = [
99
+ member
100
+ for member in self._members
101
+ if member.attached and member.probe
102
+ ]
103
+ for member in probes:
104
+ if not member.revoking:
105
+ member.revoking = True
106
+ member.probe_generation += 1
107
+ return probes
108
+
109
+ async def _finish_resize(self, capacity: int, probes: list[_Member]) -> None:
110
+ async with self._lock:
111
+ if not self._grant_paused:
112
+ await self._grant_locked()
113
+ return
114
+ probe_ids = {id(member) for member in probes}
115
+ actions = [
116
+ (member, id(member) in probe_ids)
117
+ for member in self._members
118
+ if member.attached and member.session_ready
119
+ ]
120
+ results = await asyncio.gather(
121
+ *(
122
+ member.worker._withdraw_controller_probe(member, capacity)
123
+ if has_probe
124
+ else member.worker._publish_controller_target(capacity)
125
+ for member, has_probe in actions
126
+ ),
127
+ return_exceptions=True,
128
+ )
129
+ failed = False
130
+ async with self._lock:
131
+ for (member, has_probe), result in zip(actions, results):
132
+ if result is not True:
133
+ failed = True
134
+ elif has_probe and member.probe and member.revoking:
135
+ member.probe = False
136
+ member.revoking = False
137
+ self._resize_needs_retry = failed
138
+ self._grant_paused = failed
139
+ if not failed:
140
+ await self._grant_locked()
141
+ if failed:
142
+ raise UnavailableError(
143
+ "concurrency capacity changed locally, but an old Work probe "
144
+ "or target snapshot could not be published; retry after the "
145
+ "stream reconnects"
146
+ )
147
+
148
+ async def close(self) -> None:
149
+ """Stop future grants and withdraw attached idle probes."""
150
+ self._check_loop()
151
+ async with self._resize_lock:
152
+ async with self._lock:
153
+ self._closed = True
154
+ self._grant_paused = True
155
+ probes = [member for member in self._members if member.probe]
156
+ for member in probes:
157
+ member.revoking = True
158
+ member.probe_generation += 1
159
+ results: list[object] = []
160
+ if probes:
161
+ results = await asyncio.gather(
162
+ *(
163
+ member.worker._withdraw_controller_probe(
164
+ member, 0, force_target=True
165
+ )
166
+ for member in probes
167
+ ),
168
+ return_exceptions=True,
169
+ )
170
+ async with self._lock:
171
+ for member, result in zip(probes, results):
172
+ if result is True and member.probe and member.revoking:
173
+ member.probe = False
174
+ member.revoking = False
175
+
176
+ # The following methods are Worker integration points. They are kept
177
+ # private so a controller cannot be confused with a transport-level quota.
178
+ def _check_loop(self) -> None:
179
+ loop = asyncio.get_running_loop()
180
+ if self._loop is None:
181
+ self._loop = loop
182
+ elif self._loop is not loop:
183
+ raise RuntimeError("a concurrency controller cannot cross event loops")
184
+
185
+ def _attach(self, worker: "Worker", weight: int) -> _Member:
186
+ if self._closed:
187
+ raise RuntimeError("concurrency controller is closed")
188
+ if isinstance(weight, bool) or not isinstance(weight, int) or weight < 1:
189
+ raise ValueError("weight must be a positive integer")
190
+ for member in self._members:
191
+ if member.worker is worker:
192
+ return member
193
+ member = _Member(worker=worker, weight=weight)
194
+ self._members.append(member)
195
+ return member
196
+
197
+ async def _start_member(self, member: _Member | None) -> None:
198
+ if member is None:
199
+ return
200
+ self._check_loop()
201
+ # Session admission and capacity transitions share the complete resize
202
+ # lane, including the initial WorkResult wait. A stale initial target
203
+ # therefore cannot land after a completed resize.
204
+ async with self._resize_lock:
205
+ await member.worker._publish_controller_initial(self._capacity)
206
+ async with self._lock:
207
+ if member.attached and not self._closed:
208
+ member.session_ready = True
209
+ await self._grant_locked()
210
+
211
+ async def _detach(self, member: _Member) -> None:
212
+ self._check_loop()
213
+ async with self._resize_lock:
214
+ async with self._lock:
215
+ if not member.attached:
216
+ return
217
+ member.attached = False
218
+ member.session_ready = False
219
+ needs_withdraw = member.probe and not member.revoking
220
+ if needs_withdraw:
221
+ member.revoking = True
222
+ member.probe_generation += 1
223
+ withdrawn = True
224
+ if needs_withdraw:
225
+ withdrawn = await member.worker._withdraw_controller_probe(
226
+ member, 0, force_target=True
227
+ )
228
+ async with self._lock:
229
+ if withdrawn is True and member.probe and member.revoking:
230
+ member.probe = False
231
+ member.revoking = False
232
+ await self._grant_locked()
233
+ self._prune_detached()
234
+
235
+ def _prune_detached(self) -> None:
236
+ self._members[:] = [
237
+ member
238
+ for member in self._members
239
+ if member.attached or member.running or member.probe
240
+ ]
241
+ if self._members:
242
+ self._cursor %= len(self._members)
243
+ else:
244
+ self._cursor = 0
245
+
246
+ async def _stream_lost(self, member: _Member | None) -> None:
247
+ """Return a probe whose old Work stream disappeared.
248
+
249
+ The server-side credit vanished with that stream. Running permits are
250
+ deliberately untouched; only the unused probe is made available for
251
+ republication after the replacement session registers.
252
+ """
253
+ if member is None:
254
+ return
255
+ self._check_loop()
256
+ async with self._lock:
257
+ member.session_ready = False
258
+ if member.probe:
259
+ member.probe = False
260
+ member.revoking = False
261
+ member.probe_generation += 1
262
+ await self._grant_locked()
263
+
264
+ async def _probe_consumed(self, member: _Member) -> bool:
265
+ self._check_loop()
266
+ async with self._lock:
267
+ if not member.probe:
268
+ return False
269
+ # This also handles assignment-before-revoke-result: the probe is
270
+ # promoted to running and the in-flight withdrawal must not release
271
+ # the resulting permit.
272
+ member.probe = False
273
+ member.revoking = False
274
+ member.active = True
275
+ member.running += 1
276
+ await self._grant_locked()
277
+ return True
278
+
279
+ async def _release_running(
280
+ self,
281
+ member: _Member,
282
+ *,
283
+ success: bool = False,
284
+ congestion: bool = False,
285
+ ) -> None:
286
+ self._check_loop()
287
+ # Automatic AIMD transitions and manual set_capacity calls share one
288
+ # end-to-end resize lane, including the correlated withdrawal wait.
289
+ async with self._resize_lock:
290
+ resize_capacity: int | None = None
291
+ probes: list[_Member] = []
292
+ async with self._lock:
293
+ controller = self
294
+ saturated = self._admission_count() >= self._capacity
295
+ if member.running:
296
+ member.running -= 1
297
+ if self._closed:
298
+ self._prune_detached()
299
+ return
300
+ if isinstance(controller, AdaptiveConcurrency):
301
+ if success and saturated and self._capacity > 0:
302
+ controller._successes += 1
303
+ controller._successes = min(
304
+ controller._successes, self._capacity
305
+ )
306
+ if controller._successes >= self._capacity:
307
+ controller._successes = 0
308
+ new_capacity = min(
309
+ _MAX_CAPACITY,
310
+ controller._capacity + 1,
311
+ controller.maximum,
312
+ )
313
+ if new_capacity != controller._capacity:
314
+ resize_capacity = new_capacity
315
+ probes = self._prepare_resize_locked(new_capacity)
316
+ elif congestion:
317
+ controller._successes = 0
318
+ resize_capacity = max(
319
+ controller.minimum,
320
+ int(controller._capacity * controller.mult_decrease),
321
+ )
322
+ if resize_capacity < controller._capacity:
323
+ # Stop grants and reserve every probe before releasing
324
+ # the lock; withdrawal itself happens outside it.
325
+ probes = self._prepare_resize_locked(resize_capacity)
326
+ if resize_capacity is None:
327
+ await self._grant_locked()
328
+ self._prune_detached()
329
+ if resize_capacity is not None:
330
+ await self._finish_resize(resize_capacity, probes)
331
+
332
+ def _admission_count(self) -> int:
333
+ return sum(member.running + int(member.probe) for member in self._members)
334
+
335
+ def _choose_member(self) -> _Member | None:
336
+ """Choose with smooth weighted round-robin admission.
337
+
338
+ Members whose probes have expired enter a short cooldown before the
339
+ same weighted rotation can issue a bounded discovery probe again. A
340
+ continuously consuming member therefore remains work-conserving while
341
+ an idle member is never permanently starved.
342
+ """
343
+ now = asyncio.get_running_loop().time()
344
+ eligible = [
345
+ member
346
+ for member in self._members
347
+ if member.attached
348
+ and member.session_ready
349
+ and not member.probe
350
+ and (member.active or now >= member.cooldown_until)
351
+ ]
352
+ if not eligible:
353
+ return None
354
+ total = sum(member.weight for member in eligible)
355
+ count = len(self._members)
356
+ ordered = [
357
+ self._members[(self._cursor + i) % count]
358
+ for i in range(count)
359
+ if self._members[(self._cursor + i) % count] in eligible
360
+ ]
361
+ for member in eligible:
362
+ member.deficit += member.weight
363
+ best = max(ordered, key=lambda member: member.deficit)
364
+ best.deficit -= total
365
+ self._cursor = (self._members.index(best) + 1) % count
366
+ return best
367
+
368
+ async def _grant_locked(self) -> None:
369
+ if self._closed or self._grant_paused:
370
+ return
371
+ while self._admission_count() < self._capacity:
372
+ member = self._choose_member()
373
+ if member is None:
374
+ return
375
+ member.probe = True
376
+ member.probe_generation += 1
377
+ generation = member.probe_generation
378
+ asyncio.create_task(
379
+ self._publish_probe(member, generation),
380
+ name=f"ringo-concurrency-probe-{member.worker.worker_id}",
381
+ )
382
+
383
+ async def _publish_probe(self, member: _Member, generation: int) -> None:
384
+ try:
385
+ await member.worker._publish_controller_probe(
386
+ member, self._capacity, generation
387
+ )
388
+ # An unused probe is a temporary admission reservation only. It
389
+ # must be withdrawn even when the server has no task for a member.
390
+ asyncio.create_task(
391
+ self._expire_probe(member, generation),
392
+ name=f"ringo-probe-expiry-{member.worker.worker_id}",
393
+ )
394
+ except Exception:
395
+ async with self._lock:
396
+ if member.probe and member.probe_generation == generation:
397
+ member.probe = False
398
+ member.active = False
399
+ member.cooldown_until = (
400
+ asyncio.get_running_loop().time() + _PROBE_SECONDS
401
+ )
402
+ asyncio.create_task(
403
+ self._wake_member(member),
404
+ name=f"ringo-probe-failure-{member.worker.worker_id}",
405
+ )
406
+ await self._grant_locked()
407
+
408
+ async def _expire_probe(self, member: _Member, generation: int) -> None:
409
+ await asyncio.sleep(_PROBE_SECONDS)
410
+ async with self._lock:
411
+ if (
412
+ self._closed
413
+ or not member.probe
414
+ or member.revoking
415
+ or member.probe_generation != generation
416
+ ):
417
+ return
418
+ member.revoking = True
419
+ member.active = False
420
+ member.probe_generation += 1
421
+ capacity = self._capacity
422
+ withdrawn = await member.worker._withdraw_controller_probe(member, capacity)
423
+ async with self._lock:
424
+ if withdrawn is True and member.probe and member.revoking:
425
+ member.probe = False
426
+ member.revoking = False
427
+ member.cooldown_until = (
428
+ asyncio.get_running_loop().time() + _PROBE_SECONDS
429
+ )
430
+ asyncio.create_task(
431
+ self._wake_member(member),
432
+ name=f"ringo-probe-cooldown-{member.worker.worker_id}",
433
+ )
434
+ await self._grant_locked()
435
+
436
+ async def _wake_member(self, member: _Member) -> None:
437
+ delay = max(0.0, member.cooldown_until - asyncio.get_running_loop().time())
438
+ if delay:
439
+ await asyncio.sleep(delay)
440
+ async with self._lock:
441
+ if member.attached and not member.probe and not self._closed:
442
+ await self._grant_locked()
443
+
444
+
445
+ class AdaptiveConcurrency(ResourceConcurrency):
446
+ """AIMD resource controller (additive increase, multiplicative decrease)."""
447
+
448
+ def __init__(
449
+ self,
450
+ initial: int,
451
+ *,
452
+ minimum: int = 1,
453
+ maximum: int = _MAX_CAPACITY,
454
+ ) -> None:
455
+ self._validate_capacity(minimum)
456
+ self._validate_capacity(maximum)
457
+ self._validate_capacity(initial)
458
+ if minimum > maximum:
459
+ raise ValueError("minimum must not exceed maximum")
460
+ if not minimum <= initial <= maximum:
461
+ raise ValueError("initial must be between minimum and maximum")
462
+ super().__init__(initial)
463
+ self.minimum = minimum
464
+ self.maximum = maximum
465
+ self.mult_decrease = 0.5
466
+ self._successes = 0
467
+
468
+ async def set_capacity(self, capacity: int) -> None:
469
+ self._validate_capacity(capacity)
470
+ if not self.minimum <= capacity <= self.maximum:
471
+ raise ValueError("capacity must be between minimum and maximum")
472
+ self._check_loop()
473
+ async with self._resize_lock:
474
+ async with self._lock:
475
+ self._successes = 0
476
+ probes = self._prepare_resize_locked(capacity)
477
+ await self._finish_resize(capacity, probes)
478
+
479
+
480
+ __all__ = ["AdaptiveConcurrency", "ResourceConcurrency"]