coderouter-cli 2.6.1__py3-none-any.whl → 2.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coderouter/adapters/anthropic_native.py +66 -30
- coderouter/adapters/base.py +46 -3
- coderouter/adapters/openai_compat.py +53 -11
- coderouter/config/capability_registry.py +22 -0
- coderouter/config/schemas.py +232 -0
- coderouter/guards/context_budget.py +114 -19
- coderouter/guards/continuous_probe.py +9 -1
- coderouter/guards/memory_pressure.py +16 -2
- coderouter/guards/self_healing.py +14 -0
- coderouter/ingress/anthropic_routes.py +223 -1
- coderouter/ingress/app.py +162 -1
- coderouter/ingress/launcher_routes.py +187 -11
- coderouter/ingress/openai_routes.py +77 -7
- coderouter/logging.py +29 -3
- coderouter/metrics/collector.py +454 -245
- coderouter/metrics/prometheus.py +3 -3
- coderouter/routing/capability.py +217 -0
- coderouter/routing/fallback.py +539 -27
- coderouter/state/audit_log.py +69 -7
- coderouter/state/request_log.py +70 -7
- coderouter/translation/convert.py +112 -9
- coderouter/translation/tool_repair.py +671 -50
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/METADATA +28 -9
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/RECORD +27 -27
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/WHEEL +0 -0
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/entry_points.txt +0 -0
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/licenses/LICENSE +0 -0
coderouter/metrics/collector.py
CHANGED
|
@@ -69,12 +69,53 @@ import os
|
|
|
69
69
|
import threading
|
|
70
70
|
import time
|
|
71
71
|
from collections import Counter, deque
|
|
72
|
+
from collections.abc import Callable
|
|
72
73
|
from datetime import UTC, datetime
|
|
73
74
|
from pathlib import Path
|
|
74
75
|
from typing import Any, Final
|
|
75
76
|
|
|
76
77
|
from coderouter.logging import JsonLineFormatter
|
|
77
78
|
|
|
79
|
+
# M12(1): the complete set of log-event names the collector reacts to.
|
|
80
|
+
# ``_dispatch`` tests membership here BEFORE acquiring ``self._lock`` so
|
|
81
|
+
# unrecognized (typically high-volume DEBUG) records bail out with a
|
|
82
|
+
# single frozenset lookup and never contend on the lock. Kept in lockstep
|
|
83
|
+
# with ``MetricsCollector._dispatch_table`` (built per-instance in
|
|
84
|
+
# ``__init__``); a mismatch is caught by the M12 early-exit test.
|
|
85
|
+
_KNOWN_EVENTS: Final[frozenset[str]] = frozenset(
|
|
86
|
+
{
|
|
87
|
+
"try-provider",
|
|
88
|
+
"provider-ok",
|
|
89
|
+
"provider-failed",
|
|
90
|
+
"provider-failed-midstream",
|
|
91
|
+
"skip-paid-provider",
|
|
92
|
+
"skip-unknown-provider",
|
|
93
|
+
"skip-budget-exceeded",
|
|
94
|
+
"chain-budget-exceeded",
|
|
95
|
+
"skip-memory-pressure",
|
|
96
|
+
"chain-memory-pressure-blocked",
|
|
97
|
+
"backend-health-changed",
|
|
98
|
+
"demote-unhealthy-provider",
|
|
99
|
+
"capability-degraded",
|
|
100
|
+
"output-filter-applied",
|
|
101
|
+
"chain-paid-gate-blocked",
|
|
102
|
+
"chain-uniform-auth-failure",
|
|
103
|
+
"auto-router-fallthrough",
|
|
104
|
+
"cache-observed",
|
|
105
|
+
"context-budget-warning",
|
|
106
|
+
"context-budget-trimmed",
|
|
107
|
+
"tokens-saved",
|
|
108
|
+
"drift-detected",
|
|
109
|
+
"drift-promoted",
|
|
110
|
+
"drift-reload-attempted",
|
|
111
|
+
"partial-stitch-surfaced",
|
|
112
|
+
"probe-completed",
|
|
113
|
+
"probe-round-completed",
|
|
114
|
+
"probe-capabilities-drift",
|
|
115
|
+
"coderouter-startup",
|
|
116
|
+
}
|
|
117
|
+
)
|
|
118
|
+
|
|
78
119
|
# Default ring-buffer size. Chosen to match a ~2-second refresh at 100 RPS
|
|
79
120
|
# without blowing memory; overridable via the :class:`MetricsCollector`
|
|
80
121
|
# constructor for tests.
|
|
@@ -253,6 +294,47 @@ class MetricsCollector(logging.Handler):
|
|
|
253
294
|
# YAML.
|
|
254
295
|
self._startup_info: dict[str, Any] = {}
|
|
255
296
|
|
|
297
|
+
# M12(1): event name -> bound handler method. Built once per
|
|
298
|
+
# instance (handlers are bound methods, so they must be built here
|
|
299
|
+
# rather than as a class attribute). Replaces the former
|
|
300
|
+
# ~30-branch if/elif chain in ``_dispatch``. The key set is kept
|
|
301
|
+
# in lockstep with the module-level ``_KNOWN_EVENTS`` frozenset,
|
|
302
|
+
# which ``_dispatch`` uses for a lock-free early bail-out on
|
|
303
|
+
# unrecognized (typically DEBUG) log records.
|
|
304
|
+
self._dispatch_table: dict[
|
|
305
|
+
str, Callable[[dict[str, Any], logging.LogRecord], None]
|
|
306
|
+
] = {
|
|
307
|
+
"try-provider": self._on_try_provider,
|
|
308
|
+
"provider-ok": self._on_provider_ok,
|
|
309
|
+
"provider-failed": self._on_provider_failed,
|
|
310
|
+
"provider-failed-midstream": self._on_provider_failed_midstream,
|
|
311
|
+
"skip-paid-provider": self._on_skip_paid_provider,
|
|
312
|
+
"skip-unknown-provider": self._on_skip_unknown_provider,
|
|
313
|
+
"skip-budget-exceeded": self._on_skip_budget_exceeded,
|
|
314
|
+
"chain-budget-exceeded": self._on_chain_budget_exceeded,
|
|
315
|
+
"skip-memory-pressure": self._on_skip_memory_pressure,
|
|
316
|
+
"chain-memory-pressure-blocked": self._on_chain_memory_pressure_blocked,
|
|
317
|
+
"backend-health-changed": self._on_backend_health_changed,
|
|
318
|
+
"demote-unhealthy-provider": self._on_demote_unhealthy_provider,
|
|
319
|
+
"capability-degraded": self._on_capability_degraded,
|
|
320
|
+
"output-filter-applied": self._on_output_filter_applied,
|
|
321
|
+
"chain-paid-gate-blocked": self._on_chain_paid_gate_blocked,
|
|
322
|
+
"chain-uniform-auth-failure": self._on_chain_uniform_auth_failure,
|
|
323
|
+
"auto-router-fallthrough": self._on_auto_router_fallthrough,
|
|
324
|
+
"cache-observed": self._on_cache_observed,
|
|
325
|
+
"context-budget-warning": self._on_context_budget_warning,
|
|
326
|
+
"context-budget-trimmed": self._on_context_budget_trimmed,
|
|
327
|
+
"tokens-saved": self._on_tokens_saved,
|
|
328
|
+
"drift-detected": self._on_drift_detected,
|
|
329
|
+
"drift-promoted": self._on_drift_promoted,
|
|
330
|
+
"drift-reload-attempted": self._on_drift_reload_attempted,
|
|
331
|
+
"partial-stitch-surfaced": self._on_partial_stitch_surfaced,
|
|
332
|
+
"probe-completed": self._on_probe_completed,
|
|
333
|
+
"probe-round-completed": self._on_probe_round_completed,
|
|
334
|
+
"probe-capabilities-drift": self._on_probe_capabilities_drift,
|
|
335
|
+
"coderouter-startup": self._on_coderouter_startup,
|
|
336
|
+
}
|
|
337
|
+
|
|
256
338
|
# ------------------------------------------------------------------
|
|
257
339
|
# Handler API
|
|
258
340
|
# ------------------------------------------------------------------
|
|
@@ -271,254 +353,364 @@ class MetricsCollector(logging.Handler):
|
|
|
271
353
|
self.handleError(record)
|
|
272
354
|
|
|
273
355
|
def _dispatch(self, record: logging.LogRecord) -> None:
|
|
274
|
-
"""Event name → counter/ring mutation.
|
|
356
|
+
"""Event name → counter/ring mutation.
|
|
357
|
+
|
|
358
|
+
M12(1) performance fix: the hot path is invoked for *every* log
|
|
359
|
+
record on the root logger, including high-volume DEBUG lines that
|
|
360
|
+
are never metrics events. Two cheap gates run before we touch the
|
|
361
|
+
lock:
|
|
362
|
+
|
|
363
|
+
1. ``event`` must be a ``str`` (skips records whose ``msg`` is a
|
|
364
|
+
format object / non-string).
|
|
365
|
+
2. ``event`` must be in :data:`_KNOWN_EVENTS` — a ``frozenset``
|
|
366
|
+
membership test — which lets an unrecognized event bail out
|
|
367
|
+
before acquiring ``self._lock`` and before running any of the
|
|
368
|
+
per-event handlers.
|
|
369
|
+
|
|
370
|
+
Recognized events dispatch through :data:`_DISPATCH` (a
|
|
371
|
+
``dict[str, method]`` built once per instance in ``__init__``),
|
|
372
|
+
replacing the former ~30-branch ``if/elif`` chain. Behavior is
|
|
373
|
+
identical to the chain — each handler body is the verbatim former
|
|
374
|
+
branch — but recognized-event dispatch is now O(1) and unknown
|
|
375
|
+
events never take the lock.
|
|
376
|
+
"""
|
|
275
377
|
event = record.msg
|
|
276
|
-
if not isinstance(event, str):
|
|
378
|
+
if not isinstance(event, str) or event not in _KNOWN_EVENTS:
|
|
379
|
+
return
|
|
380
|
+
handler = self._dispatch_table.get(event)
|
|
381
|
+
if handler is None: # pragma: no cover - _KNOWN_EVENTS/table in sync
|
|
277
382
|
return
|
|
278
383
|
extras = record.__dict__
|
|
279
384
|
with self._lock:
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
385
|
+
handler(extras, record)
|
|
386
|
+
|
|
387
|
+
# ------------------------------------------------------------------
|
|
388
|
+
# Per-event handlers (M12: extracted verbatim from the former
|
|
389
|
+
# if/elif chain in _dispatch). Each is called under ``self._lock``.
|
|
390
|
+
# ------------------------------------------------------------------
|
|
391
|
+
|
|
392
|
+
def _on_try_provider(
|
|
393
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
394
|
+
) -> None:
|
|
395
|
+
self._requests_total += 1
|
|
396
|
+
provider = _str(extras.get("provider"))
|
|
397
|
+
self._provider_attempts[provider] += 1
|
|
398
|
+
self._push_recent("try-provider", extras, record)
|
|
399
|
+
|
|
400
|
+
def _on_provider_ok(
|
|
401
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
402
|
+
) -> None:
|
|
403
|
+
provider = _str(extras.get("provider"))
|
|
404
|
+
self._provider_outcomes.setdefault(provider, Counter())["ok"] += 1
|
|
405
|
+
self._push_recent("provider-ok", extras, record)
|
|
406
|
+
|
|
407
|
+
def _on_provider_failed(
|
|
408
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
409
|
+
) -> None:
|
|
410
|
+
provider = _str(extras.get("provider"))
|
|
411
|
+
self._provider_outcomes.setdefault(provider, Counter())["failed"] += 1
|
|
412
|
+
self._last_error[provider] = _make_last_error(extras, record)
|
|
413
|
+
self._push_recent("provider-failed", extras, record)
|
|
414
|
+
|
|
415
|
+
def _on_provider_failed_midstream(
|
|
416
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
417
|
+
) -> None:
|
|
418
|
+
provider = _str(extras.get("provider"))
|
|
419
|
+
self._provider_outcomes.setdefault(provider, Counter())[
|
|
420
|
+
"failed_midstream"
|
|
421
|
+
] += 1
|
|
422
|
+
self._last_error[provider] = _make_last_error(extras, record)
|
|
423
|
+
self._push_recent("provider-failed-midstream", extras, record)
|
|
424
|
+
|
|
425
|
+
def _on_skip_paid_provider(
|
|
426
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
427
|
+
) -> None:
|
|
428
|
+
provider = _str(extras.get("provider"))
|
|
429
|
+
self._provider_skipped_paid[provider] += 1
|
|
430
|
+
|
|
431
|
+
def _on_skip_unknown_provider(
|
|
432
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
433
|
+
) -> None:
|
|
434
|
+
provider = _str(extras.get("provider"))
|
|
435
|
+
self._provider_skipped_unknown[provider] += 1
|
|
436
|
+
|
|
437
|
+
def _on_skip_budget_exceeded(
|
|
438
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
439
|
+
) -> None:
|
|
440
|
+
# v1.10: per-provider budget-gate filter count.
|
|
441
|
+
provider = _str(extras.get("provider"))
|
|
442
|
+
self._provider_skipped_budget[provider] += 1
|
|
443
|
+
|
|
444
|
+
def _on_chain_budget_exceeded(
|
|
445
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
446
|
+
) -> None:
|
|
447
|
+
# v1.10: chain-level aggregate (whole chain blocked by the
|
|
448
|
+
# budget gate). Symmetric with ``chain-paid-gate-blocked``.
|
|
449
|
+
self._chain_budget_exceeded_total += 1
|
|
450
|
+
|
|
451
|
+
def _on_skip_memory_pressure(
|
|
452
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
453
|
+
) -> None:
|
|
454
|
+
# v1.9-E phase 2 (L2): per-provider OOM-cooldown filter count.
|
|
455
|
+
# Same shape as the paid / budget skip counters.
|
|
456
|
+
provider = _str(extras.get("provider"))
|
|
457
|
+
self._provider_skipped_memory_pressure[provider] += 1
|
|
458
|
+
|
|
459
|
+
def _on_chain_memory_pressure_blocked(
|
|
460
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
461
|
+
) -> None:
|
|
462
|
+
# v1.9-E phase 2 (L2): chain-level aggregate.
|
|
463
|
+
self._chain_memory_pressure_blocked_total += 1
|
|
464
|
+
|
|
465
|
+
def _on_backend_health_changed(
|
|
466
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
467
|
+
) -> None:
|
|
468
|
+
# v1.9-E phase 2 (L5): per-provider state-transition counter,
|
|
469
|
+
# keyed on the new state. Lets dashboards render a "providers
|
|
470
|
+
# degraded right now" panel by comparing transition counts to
|
|
471
|
+
# recovery counts.
|
|
472
|
+
provider = _str(extras.get("provider"))
|
|
473
|
+
new_state = _str(extras.get("new_state"))
|
|
474
|
+
if new_state:
|
|
475
|
+
self._backend_health_transitions.setdefault(
|
|
476
|
+
provider, Counter()
|
|
477
|
+
)[new_state] += 1
|
|
478
|
+
|
|
479
|
+
def _on_demote_unhealthy_provider(
|
|
480
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
481
|
+
) -> None:
|
|
482
|
+
# v1.9-E phase 2 (L5): per-provider demote count.
|
|
483
|
+
provider = _str(extras.get("provider"))
|
|
484
|
+
self._provider_demoted_unhealthy[provider] += 1
|
|
485
|
+
|
|
486
|
+
def _on_capability_degraded(
|
|
487
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
488
|
+
) -> None:
|
|
489
|
+
dropped = extras.get("dropped") or []
|
|
490
|
+
if isinstance(dropped, list):
|
|
491
|
+
for cap in dropped:
|
|
492
|
+
if isinstance(cap, str):
|
|
493
|
+
self._capability_degraded[cap] += 1
|
|
494
|
+
|
|
495
|
+
def _on_output_filter_applied(
|
|
496
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
497
|
+
) -> None:
|
|
498
|
+
filters = extras.get("filters") or []
|
|
499
|
+
if isinstance(filters, list):
|
|
500
|
+
for name in filters:
|
|
501
|
+
if isinstance(name, str):
|
|
502
|
+
self._output_filter_applied[name] += 1
|
|
503
|
+
|
|
504
|
+
def _on_chain_paid_gate_blocked(
|
|
505
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
506
|
+
) -> None:
|
|
507
|
+
self._chain_paid_gate_blocked_total += 1
|
|
508
|
+
|
|
509
|
+
def _on_chain_uniform_auth_failure(
|
|
510
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
511
|
+
) -> None:
|
|
512
|
+
self._chain_uniform_auth_failure_total += 1
|
|
513
|
+
|
|
514
|
+
def _on_auto_router_fallthrough(
|
|
515
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
516
|
+
) -> None:
|
|
517
|
+
# Every call into ``classify()`` that exits via the default-rule
|
|
518
|
+
# branch (no user/bundled rule matched, or
|
|
519
|
+
# ``auto_router.disabled: true``) bumps this counter.
|
|
520
|
+
self._auto_router_fallthrough_total += 1
|
|
521
|
+
|
|
522
|
+
def _on_cache_observed(
|
|
523
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
524
|
+
) -> None:
|
|
525
|
+
# v1.9-A: per-provider cache token + 4-class outcome.
|
|
526
|
+
# Defensive int-coerce — log extras are typed via
|
|
527
|
+
# CacheObservedPayload at the source but the handler
|
|
528
|
+
# contract still lets us receive anything, and we never
|
|
529
|
+
# want a malformed log line to crash the metrics tap.
|
|
530
|
+
provider = _str(extras.get("provider"))
|
|
531
|
+
read_raw = extras.get("cache_read_input_tokens", 0)
|
|
532
|
+
creation_raw = extras.get("cache_creation_input_tokens", 0)
|
|
533
|
+
read = read_raw if isinstance(read_raw, int) else 0
|
|
534
|
+
creation = creation_raw if isinstance(creation_raw, int) else 0
|
|
535
|
+
outcome = _str(extras.get("outcome"))
|
|
536
|
+
self._cache_read_tokens[provider] += read
|
|
537
|
+
self._cache_creation_tokens[provider] += creation
|
|
538
|
+
self._cache_read_tokens_total += read
|
|
539
|
+
self._cache_creation_tokens_total += creation
|
|
540
|
+
if outcome:
|
|
541
|
+
self._cache_outcomes.setdefault(provider, Counter())[outcome] += 1
|
|
542
|
+
|
|
543
|
+
# v1.9-D: cost aggregation. Same defensive coercion —
|
|
544
|
+
# malformed cost values default to 0.0 rather than
|
|
545
|
+
# crashing the metrics handler. The fields are typed
|
|
546
|
+
# ``float`` in CacheObservedPayload but JSON parsers
|
|
547
|
+
# downstream can sometimes hand us ``int`` for whole
|
|
548
|
+
# numbers, so we accept both.
|
|
549
|
+
cost_usd_raw = extras.get("cost_usd", 0.0)
|
|
550
|
+
savings_usd_raw = extras.get("cost_savings_usd", 0.0)
|
|
551
|
+
cost_usd = (
|
|
552
|
+
float(cost_usd_raw)
|
|
553
|
+
if isinstance(cost_usd_raw, int | float)
|
|
554
|
+
else 0.0
|
|
555
|
+
)
|
|
556
|
+
savings_usd = (
|
|
557
|
+
float(savings_usd_raw)
|
|
558
|
+
if isinstance(savings_usd_raw, int | float)
|
|
559
|
+
else 0.0
|
|
560
|
+
)
|
|
561
|
+
if cost_usd > 0.0:
|
|
562
|
+
self._cost_total_usd[provider] = (
|
|
563
|
+
self._cost_total_usd.get(provider, 0.0) + cost_usd
|
|
564
|
+
)
|
|
565
|
+
self._cost_total_usd_aggregate += cost_usd
|
|
566
|
+
if savings_usd > 0.0:
|
|
567
|
+
self._cost_savings_usd[provider] = (
|
|
568
|
+
self._cost_savings_usd.get(provider, 0.0) + savings_usd
|
|
569
|
+
)
|
|
570
|
+
self._cost_savings_usd_aggregate += savings_usd
|
|
571
|
+
|
|
572
|
+
# v2.6: language-tax spend. Same defensive coercion as the
|
|
573
|
+
# cost fields; defaults to 0.0 for pre-v2.6 log lines and
|
|
574
|
+
# English/code traffic, so the aggregate only moves on
|
|
575
|
+
# CJK-heavy requests against a tokenizer-configured provider.
|
|
576
|
+
lt_usd_raw = extras.get("language_tax_usd", 0.0)
|
|
577
|
+
lt_usd = (
|
|
578
|
+
float(lt_usd_raw)
|
|
579
|
+
if isinstance(lt_usd_raw, int | float)
|
|
580
|
+
else 0.0
|
|
581
|
+
)
|
|
582
|
+
if lt_usd > 0.0:
|
|
583
|
+
self._language_tax_usd[provider] = (
|
|
584
|
+
self._language_tax_usd.get(provider, 0.0) + lt_usd
|
|
585
|
+
)
|
|
586
|
+
self._language_tax_usd_aggregate += lt_usd
|
|
587
|
+
|
|
588
|
+
def _on_context_budget_warning(
|
|
589
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
590
|
+
) -> None:
|
|
591
|
+
# v2.0-F (L1): context usage exceeded the warn threshold.
|
|
592
|
+
# Track per-profile and aggregate, plus latest ratio gauge.
|
|
593
|
+
profile = _str(extras.get("profile"))
|
|
594
|
+
self._context_budget_warnings_total += 1
|
|
595
|
+
self._context_budget_warnings_by_profile[profile] += 1
|
|
596
|
+
ratio_raw = extras.get("usage_ratio")
|
|
597
|
+
if isinstance(ratio_raw, int | float):
|
|
598
|
+
self._context_budget_latest_ratio[profile] = float(ratio_raw)
|
|
599
|
+
|
|
600
|
+
def _on_context_budget_trimmed(
|
|
601
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
602
|
+
) -> None:
|
|
603
|
+
# v2.0-F (L1): messages were removed to fit the budget.
|
|
604
|
+
profile = _str(extras.get("profile"))
|
|
605
|
+
self._context_budget_trims_total += 1
|
|
606
|
+
self._context_budget_trims_by_profile[profile] += 1
|
|
607
|
+
# token-savings: the trim already carries the estimated
|
|
608
|
+
# before/after token counts, so fold the delta into the
|
|
609
|
+
# shared savings buckets (mechanism="trim").
|
|
610
|
+
before = extras.get("estimated_tokens_before")
|
|
611
|
+
after = extras.get("estimated_tokens_after")
|
|
612
|
+
if isinstance(before, int | float) and isinstance(after, int | float):
|
|
613
|
+
saved = max(0, int(before) - int(after))
|
|
614
|
+
if saved:
|
|
615
|
+
self._tokens_saved_total += saved
|
|
616
|
+
self._tokens_saved_by_mechanism["trim"] += saved
|
|
617
|
+
|
|
618
|
+
def _on_tokens_saved(
|
|
619
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
620
|
+
) -> None:
|
|
621
|
+
# Neutral token-savings event emitted by non-core reduction
|
|
622
|
+
# mechanisms (e.g. the compress plugin). Carries a
|
|
623
|
+
# ``mechanism`` label plus either a precomputed
|
|
624
|
+
# ``tokens_saved`` or a ``tokens_before`` / ``tokens_after``
|
|
625
|
+
# pair. Negative deltas clamp to zero. No core import needed
|
|
626
|
+
# by the emitter — it just logs this record name.
|
|
627
|
+
mechanism = _str(extras.get("mechanism")) or "unknown"
|
|
628
|
+
saved_raw = extras.get("tokens_saved")
|
|
629
|
+
before = extras.get("tokens_before")
|
|
630
|
+
after = extras.get("tokens_after")
|
|
631
|
+
if isinstance(saved_raw, int | float):
|
|
632
|
+
saved = max(0, int(saved_raw))
|
|
633
|
+
elif isinstance(before, int | float) and isinstance(after, int | float):
|
|
634
|
+
saved = max(0, int(before) - int(after))
|
|
635
|
+
else:
|
|
636
|
+
saved = 0
|
|
637
|
+
if saved:
|
|
638
|
+
self._tokens_saved_total += saved
|
|
639
|
+
self._tokens_saved_by_mechanism[mechanism] += saved
|
|
640
|
+
self._push_recent("tokens-saved", extras, record)
|
|
641
|
+
|
|
642
|
+
def _on_drift_detected(
|
|
643
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
644
|
+
) -> None:
|
|
645
|
+
# v2.0-G (L4): drift detection fired.
|
|
646
|
+
provider = _str(extras.get("provider"))
|
|
647
|
+
self._drift_detected_total += 1
|
|
648
|
+
self._drift_detected_by_provider[provider] += 1
|
|
649
|
+
self._push_recent("drift-detected", extras, record)
|
|
650
|
+
|
|
651
|
+
def _on_drift_promoted(
|
|
652
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
653
|
+
) -> None:
|
|
654
|
+
# v2.0-G (L4): drifted provider was demoted.
|
|
655
|
+
self._drift_promoted_total += 1
|
|
656
|
+
self._push_recent("drift-promoted", extras, record)
|
|
657
|
+
|
|
658
|
+
def _on_drift_reload_attempted(
|
|
659
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
660
|
+
) -> None:
|
|
661
|
+
# v2.0-G (L4): Ollama KV cache flush attempted.
|
|
662
|
+
self._drift_reload_total += 1
|
|
663
|
+
if extras.get("success"):
|
|
664
|
+
self._drift_reload_success_total += 1
|
|
665
|
+
|
|
666
|
+
def _on_partial_stitch_surfaced(
|
|
667
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
668
|
+
) -> None:
|
|
669
|
+
# v2.0-H (L6): mid-stream failure gracefully surfaced.
|
|
670
|
+
self._partial_stitch_surfaced_total += 1
|
|
671
|
+
self._push_recent("partial-stitch-surfaced", extras, record)
|
|
672
|
+
|
|
673
|
+
def _on_probe_completed(
|
|
674
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
675
|
+
) -> None:
|
|
676
|
+
# v2.0-I: per-provider probe outcome.
|
|
677
|
+
provider = _str(extras.get("provider"))
|
|
678
|
+
self._probe_total[provider] += 1
|
|
679
|
+
if extras.get("success"):
|
|
680
|
+
self._probe_success[provider] += 1
|
|
681
|
+
else:
|
|
682
|
+
self._probe_failure[provider] += 1
|
|
683
|
+
latency_raw = extras.get("latency_ms")
|
|
684
|
+
if isinstance(latency_raw, int | float):
|
|
685
|
+
self._probe_latency_ms[provider] = float(latency_raw)
|
|
686
|
+
|
|
687
|
+
def _on_probe_round_completed(
|
|
688
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
689
|
+
) -> None:
|
|
690
|
+
# v2.0-I: round counter for the dashboard.
|
|
691
|
+
self._probe_rounds_total += 1
|
|
692
|
+
|
|
693
|
+
def _on_probe_capabilities_drift(
|
|
694
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
695
|
+
) -> None:
|
|
696
|
+
# v2.0-I: model mismatch detected by probe.
|
|
697
|
+
provider = _str(extras.get("provider"))
|
|
698
|
+
self._probe_drift_detected[provider] += 1
|
|
699
|
+
|
|
700
|
+
def _on_coderouter_startup(
|
|
701
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
702
|
+
) -> None:
|
|
703
|
+
# Snapshot a subset — startup payload contains lists that are
|
|
704
|
+
# safe to surface to /metrics.json. Version / providers /
|
|
705
|
+
# profiles / default_profile is all the dashboard needs.
|
|
706
|
+
self._startup_info = {
|
|
707
|
+
"version": _str(extras.get("version")),
|
|
708
|
+
"providers": list(extras.get("providers") or []),
|
|
709
|
+
"profiles": list(extras.get("profiles") or []),
|
|
710
|
+
"default_profile": _str(extras.get("default_profile")),
|
|
711
|
+
"allow_paid": bool(extras.get("allow_paid")),
|
|
712
|
+
"mode_source": _str(extras.get("mode_source")),
|
|
713
|
+
}
|
|
522
714
|
|
|
523
715
|
def _push_recent(
|
|
524
716
|
self, event: str, extras: dict[str, Any], record: logging.LogRecord
|
|
@@ -730,6 +922,12 @@ class MetricsCollector(logging.Handler):
|
|
|
730
922
|
"cost_savings_usd": dict(self._cost_savings_usd),
|
|
731
923
|
"cost_total_usd_aggregate": self._cost_total_usd_aggregate,
|
|
732
924
|
"cost_savings_usd_aggregate": self._cost_savings_usd_aggregate,
|
|
925
|
+
# v2.6: language-tax spend. M5 fix — load_state already
|
|
926
|
+
# reads these two keys, so save_state must emit them or the
|
|
927
|
+
# CJK language-tax running totals silently zero-reset on
|
|
928
|
+
# every restart.
|
|
929
|
+
"language_tax_usd": dict(self._language_tax_usd),
|
|
930
|
+
"language_tax_usd_aggregate": self._language_tax_usd_aggregate,
|
|
733
931
|
"chain_paid_gate_blocked_total": self._chain_paid_gate_blocked_total,
|
|
734
932
|
"chain_budget_exceeded_total": self._chain_budget_exceeded_total,
|
|
735
933
|
"chain_memory_pressure_blocked_total": self._chain_memory_pressure_blocked_total,
|
|
@@ -896,10 +1094,21 @@ def install_collector(*, ring_size: int = _DEFAULT_RING_SIZE) -> MetricsCollecto
|
|
|
896
1094
|
"""
|
|
897
1095
|
global _collector
|
|
898
1096
|
with _collector_lock:
|
|
1097
|
+
root = logging.getLogger()
|
|
899
1098
|
if _collector is None:
|
|
900
1099
|
_collector = MetricsCollector(ring_size=ring_size)
|
|
901
|
-
|
|
1100
|
+
root.addHandler(_collector)
|
|
902
1101
|
_install_jsonl_mirror()
|
|
1102
|
+
elif _collector not in root.handlers:
|
|
1103
|
+
# M4: the collector instance already exists but was detached
|
|
1104
|
+
# from the root logger — most commonly because a second
|
|
1105
|
+
# create_app() in the same process re-ran configure_logging.
|
|
1106
|
+
# Historically configure_logging removed every root handler;
|
|
1107
|
+
# even with that fixed, a caller that manually cleared the
|
|
1108
|
+
# root handlers would leave the collector orphaned and metrics
|
|
1109
|
+
# would silently stop. Re-attach the existing instance so its
|
|
1110
|
+
# accumulated counters survive the reconfigure.
|
|
1111
|
+
root.addHandler(_collector)
|
|
903
1112
|
return _collector
|
|
904
1113
|
|
|
905
1114
|
|