risicare 0.9.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +254 -0
  2. package/README.md +194 -58
  3. package/dist/frameworks/instructor.cjs +383 -117
  4. package/dist/frameworks/instructor.cjs.map +1 -1
  5. package/dist/frameworks/instructor.js +383 -117
  6. package/dist/frameworks/instructor.js.map +1 -1
  7. package/dist/frameworks/langchain.cjs +396 -130
  8. package/dist/frameworks/langchain.cjs.map +1 -1
  9. package/dist/frameworks/langchain.js +396 -130
  10. package/dist/frameworks/langchain.js.map +1 -1
  11. package/dist/frameworks/langgraph.cjs +396 -130
  12. package/dist/frameworks/langgraph.cjs.map +1 -1
  13. package/dist/frameworks/langgraph.js +396 -130
  14. package/dist/frameworks/langgraph.js.map +1 -1
  15. package/dist/frameworks/llamaindex.cjs +396 -130
  16. package/dist/frameworks/llamaindex.cjs.map +1 -1
  17. package/dist/frameworks/llamaindex.js +396 -130
  18. package/dist/frameworks/llamaindex.js.map +1 -1
  19. package/dist/index.cjs +1136 -240
  20. package/dist/index.cjs.map +1 -1
  21. package/dist/index.d.cts +151 -26
  22. package/dist/index.d.ts +151 -26
  23. package/dist/index.js +1132 -240
  24. package/dist/index.js.map +1 -1
  25. package/dist/providers/anthropic/index.cjs +467 -137
  26. package/dist/providers/anthropic/index.cjs.map +1 -1
  27. package/dist/providers/anthropic/index.js +467 -137
  28. package/dist/providers/anthropic/index.js.map +1 -1
  29. package/dist/providers/bedrock/index.cjs +409 -128
  30. package/dist/providers/bedrock/index.cjs.map +1 -1
  31. package/dist/providers/bedrock/index.js +409 -128
  32. package/dist/providers/bedrock/index.js.map +1 -1
  33. package/dist/providers/cerebras/index.cjs +414 -133
  34. package/dist/providers/cerebras/index.cjs.map +1 -1
  35. package/dist/providers/cerebras/index.js +414 -133
  36. package/dist/providers/cerebras/index.js.map +1 -1
  37. package/dist/providers/cohere/index.cjs +414 -133
  38. package/dist/providers/cohere/index.cjs.map +1 -1
  39. package/dist/providers/cohere/index.js +414 -133
  40. package/dist/providers/cohere/index.js.map +1 -1
  41. package/dist/providers/google/index.cjs +414 -133
  42. package/dist/providers/google/index.cjs.map +1 -1
  43. package/dist/providers/google/index.js +414 -133
  44. package/dist/providers/google/index.js.map +1 -1
  45. package/dist/providers/groq/index.cjs +414 -133
  46. package/dist/providers/groq/index.cjs.map +1 -1
  47. package/dist/providers/groq/index.js +414 -133
  48. package/dist/providers/groq/index.js.map +1 -1
  49. package/dist/providers/huggingface/index.cjs +414 -133
  50. package/dist/providers/huggingface/index.cjs.map +1 -1
  51. package/dist/providers/huggingface/index.js +414 -133
  52. package/dist/providers/huggingface/index.js.map +1 -1
  53. package/dist/providers/mistral/index.cjs +414 -133
  54. package/dist/providers/mistral/index.cjs.map +1 -1
  55. package/dist/providers/mistral/index.js +414 -133
  56. package/dist/providers/mistral/index.js.map +1 -1
  57. package/dist/providers/ollama/index.cjs +414 -133
  58. package/dist/providers/ollama/index.cjs.map +1 -1
  59. package/dist/providers/ollama/index.js +414 -133
  60. package/dist/providers/ollama/index.js.map +1 -1
  61. package/dist/providers/openai/index.cjs +452 -135
  62. package/dist/providers/openai/index.cjs.map +1 -1
  63. package/dist/providers/openai/index.js +452 -135
  64. package/dist/providers/openai/index.js.map +1 -1
  65. package/dist/providers/together/index.cjs +414 -133
  66. package/dist/providers/together/index.cjs.map +1 -1
  67. package/dist/providers/together/index.js +414 -133
  68. package/dist/providers/together/index.js.map +1 -1
  69. package/dist/providers/vercel-ai/index.cjs +398 -132
  70. package/dist/providers/vercel-ai/index.cjs.map +1 -1
  71. package/dist/providers/vercel-ai/index.js +398 -132
  72. package/dist/providers/vercel-ai/index.js.map +1 -1
  73. package/package.json +1 -1
@@ -8,6 +8,35 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
8
8
  // src/client.ts
9
9
  import { createHash as createHash2 } from "crypto";
10
10
 
11
+ // src/exporters/retry.ts
12
+ var RETRY_STATUSES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
13
+ var TOO_LARGE_STATUS = 413;
14
+ var RETRY_BACKOFF_BASE_S = 1;
15
+ var RETRY_BACKOFF_CAP_S = 30;
16
+ var DEFAULT_RETRY_HORIZON_S = 300;
17
+ var DEFAULT_SHUTDOWN_TIMEOUT_S = 10;
18
+ var RETRY_WARN_AFTER_ATTEMPTS = 3;
19
+ var jitter = { fraction: () => Math.random() };
20
+ function backoffDelayS(retryNumber, fraction) {
21
+ const exponent = Math.min(Math.max(retryNumber, 0), 62);
22
+ const ceiling = Math.min(RETRY_BACKOFF_CAP_S, RETRY_BACKOFF_BASE_S * 2 ** exponent);
23
+ return Math.min(Math.max(fraction, 0), 1) * ceiling;
24
+ }
25
+ var SECONDS = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
26
+ function parseRetryAfterS(value, nowMs = Date.now()) {
27
+ if (value === null || value === void 0) return null;
28
+ const text = String(value).trim();
29
+ if (text === "") return null;
30
+ if (SECONDS.test(text)) {
31
+ const seconds = Number(text);
32
+ return Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
33
+ }
34
+ if (!/[A-Za-z]/.test(text)) return null;
35
+ const when = Date.parse(text);
36
+ if (!Number.isFinite(when)) return null;
37
+ return Math.max(0, (when - nowMs) / 1e3);
38
+ }
39
+
11
40
  // src/globals.ts
12
41
  import { AsyncLocalStorage } from "async_hooks";
13
42
  var G = globalThis;
@@ -108,11 +137,70 @@ function hasCredentials(url) {
108
137
  }
109
138
  }
110
139
 
140
+ // src/masking.ts
141
+ var CONTENT_IN_JSON_OBJECT = /* @__PURE__ */ new Map([
142
+ [
143
+ "llm.invocation_parameters",
144
+ /* @__PURE__ */ new Set([
145
+ "additionalinstructions",
146
+ // OpenAI assistants run
147
+ "chathistory",
148
+ // Cohere
149
+ "contents",
150
+ // Gemini
151
+ "documents",
152
+ // Cohere RAG
153
+ "functions",
154
+ "input",
155
+ "inputtext",
156
+ // Bedrock Titan
157
+ "instructions",
158
+ "message",
159
+ // Cohere
160
+ "messages",
161
+ "preamble",
162
+ // Cohere
163
+ "prediction",
164
+ // OpenAI predicted output
165
+ "prompt",
166
+ "system",
167
+ // Anthropic
168
+ "systeminstruction",
169
+ // Gemini
170
+ "texts",
171
+ // Cohere embed
172
+ "toolconfig",
173
+ // Gemini
174
+ "tools"
175
+ ])
176
+ ]
177
+ ]);
178
+ var VALUE_RULE_PREFIXES = new Set(
179
+ [...CONTENT_IN_JSON_OBJECT.keys()].flatMap((key) => {
180
+ const segments = key.split(".");
181
+ return segments.slice(1).map((_, i) => segments.slice(0, i + 1).join("."));
182
+ })
183
+ );
184
+
111
185
  // src/exporters/batch.ts
186
+ var SHUTDOWN_BUDGET_MS = DEFAULT_SHUTDOWN_TIMEOUT_S * 1e3;
112
187
  var MAX_TIMEOUT_MS = 2 ** 31 - 1;
113
188
  var MAX_BATCH_BYTES = 8 * 1024 * 1024;
114
189
  var utf8ByteLength = typeof Buffer !== "undefined" && typeof Buffer.byteLength === "function" ? (s) => Buffer.byteLength(s, "utf8") : (s) => s.length * 3;
115
190
 
191
+ // src/context/storage.ts
192
+ function storage() {
193
+ return getContextStorage();
194
+ }
195
+ function getContext() {
196
+ return storage().getStore() ?? {};
197
+ }
198
+ function runWithContext(overrides, fn) {
199
+ const parent = getContext();
200
+ const merged = { ...parent, ...overrides };
201
+ return storage().run(merged, fn);
202
+ }
203
+
116
204
  // src/ids.ts
117
205
  import { randomBytes } from "crypto";
118
206
 
@@ -243,21 +331,8 @@ var SUPPLEMENTAL_RULES = [
243
331
  { re: new RegExp("json|syntaxerror|unexpected token|decodeerror", "u"), code: "PERCEPTION.PARSING.JSON_INVALID" }
244
332
  ];
245
333
 
246
- // src/context/storage.ts
247
- function storage() {
248
- return getContextStorage();
249
- }
250
- function getContext() {
251
- return storage().getStore() ?? {};
252
- }
253
- function runWithContext(overrides, fn) {
254
- const parent = getContext();
255
- const merged = { ...parent, ...overrides };
256
- return storage().run(merged, fn);
257
- }
258
-
259
334
  // src/version.ts
260
- var SDK_VERSION = "0.9.2";
335
+ var SDK_VERSION = "0.11.0";
261
336
 
262
337
  // src/exporters/quota.ts
263
338
  var QUOTA_REASON = "monthly_quota_exceeded";
@@ -365,12 +440,18 @@ var QuotaPause = class {
365
440
 
366
441
  // src/exporters/http.ts
367
442
  var CAPACITY_REFUSAL_STATUSES = /* @__PURE__ */ new Set([429, 503]);
368
- var RETRY_AFTER_CEILING_MS = 1e4;
369
- var REFUSAL_BACKOFF_BASE_MS = 1e3;
370
- var REFUSAL_NO_HINT = -1;
371
443
  var MISROUTE_WARN_WINDOW_MS = 3e5;
372
444
  var AUTH_REFUSAL_STATUSES = /* @__PURE__ */ new Set([401, 403]);
373
445
  var FAILURE_WARN_WINDOW_MS = 3e5;
446
+ var MAX_TIMER_MS = 2 ** 31 - 1;
447
+ function validHorizonS(value) {
448
+ if (value === void 0) return DEFAULT_RETRY_HORIZON_S;
449
+ if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
450
+ warn(
451
+ `Risicare: retryHorizonS=${String(value)} is not a finite number of 0 or more; using the default, ${DEFAULT_RETRY_HORIZON_S} s.`
452
+ );
453
+ return DEFAULT_RETRY_HORIZON_S;
454
+ }
374
455
  var HttpExporter = class {
375
456
  name = "http";
376
457
  _endpoint;
@@ -380,20 +461,28 @@ var HttpExporter = class {
380
461
  _timeoutMs;
381
462
  _maxRetries;
382
463
  _compress;
464
+ _retryHorizonS;
465
+ /** Total HTTP requests that this exporter sent (each retry and each half of a split counts). */
466
+ attempts = 0;
383
467
  // Circuit breaker (CON-201 / B-16). Threshold and cooldown sized for
384
- // a "30-second prod restart" scenario: with default 1s batch flush a
385
- // 5-failure threshold opens the circuit ~5s into an outage and the 60s
386
- // cooldown gives the gateway time to come back before the half-open
387
- // probe fires. Lower numbers (was 3@30s) tripped on transient blips.
468
+ // a "30-second prod restart" scenario. Since W2-1c only answers that a retry
469
+ // cannot change move it (see the block above).
388
470
  _consecutiveFailures = 0;
389
471
  _lastMisrouteWarnAt = -Infinity;
390
472
  _lastFailureWarnAt = -Infinity;
473
+ _lastExhaustedWarnAt = -Infinity;
474
+ /** "log once" (W2-1c): the WARNING for a `too_large` drop is written one time. */
475
+ _tooLargeWarned = false;
391
476
  /**
392
477
  * The kind of failure that the last outage WARNING named ('auth', 'http', 'none',
393
478
  * 'unknown'). `null` after a success. A new kind in the same outage writes one more
394
479
  * WARNING, also inside the window.
395
480
  */
396
481
  _failureKind = null;
482
+ /** `endRetries()` was called, and `shutdown()` has not run since. */
483
+ _noWait = false;
484
+ /** The wake-ups of the backoff waits in progress. `endRetries()` calls each one. */
485
+ _waits = /* @__PURE__ */ new Set();
397
486
  /** The endpoint as a log line may show it: no userinfo, no query (see `safeUrl`). */
398
487
  _safeEndpoint;
399
488
  /** The endpoint holds userinfo. `fetch` refuses such a URL. */
@@ -404,6 +493,8 @@ var HttpExporter = class {
404
493
  _acks = /* @__PURE__ */ new WeakMap();
405
494
  /** The requests that each finished call sent, by its batch. Read once by `takeAttempts(spans)`. */
406
495
  _requests = /* @__PURE__ */ new WeakMap();
496
+ /** The drop reason of each failed call, by its batch. Read once by `takeDropReason(spans)`. */
497
+ _dropReasons = /* @__PURE__ */ new WeakMap();
407
498
  /** The batch processor dropped span(s) from a full queue in the quota pause (S-1). */
408
499
  noteQuotaDrop(count = 1) {
409
500
  this._quota.addDrops(count);
@@ -428,8 +519,16 @@ var HttpExporter = class {
428
519
  this._projectId = options.projectId;
429
520
  this._environment = options.environment;
430
521
  this._timeoutMs = options.timeoutMs ?? 5e3;
431
- this._maxRetries = options.maxRetries ?? 3;
522
+ let maxRetries = options.maxRetries;
523
+ if (maxRetries !== void 0 && !(Number.isInteger(maxRetries) && maxRetries >= 1)) {
524
+ warn(
525
+ `Risicare: maxRetries=${String(maxRetries)} is not a whole number of 1 or more; the exporter uses no count limit (the retry horizon is the limit).`
526
+ );
527
+ maxRetries = void 0;
528
+ }
529
+ this._maxRetries = maxRetries;
432
530
  this._compress = options.compress ?? false;
531
+ this._retryHorizonS = validHorizonS(options.retryHorizonS);
433
532
  this._safeEndpoint = safeUrl(this._endpoint);
434
533
  this._endpointHasCredentials = hasCredentials(this._endpoint);
435
534
  }
@@ -451,16 +550,18 @@ var HttpExporter = class {
451
550
  [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
452
551
  return `HttpExporter { endpoint: '${this._safeEndpoint}' }`;
453
552
  }
454
- /** The most requests that one `export()` call sends (`maxRetries`). */
553
+ /**
554
+ * The most attempts for one request body (`maxRetries`). `undefined` when no count is set:
555
+ * the retry horizon is the limit.
556
+ */
455
557
  get attemptsPerExport() {
456
558
  return this._maxRetries;
457
559
  }
458
560
  /**
459
561
  * Returns the number of HTTP requests that the `export()` call for `spans` sent, and clears
460
- * it. This is the number that was sent, not the most that the call may send
461
- * (`attemptsPerExport`): a 401 or 403 stops the loop after one request, and an open circuit
462
- * breaker sends none. Returns `null` when no call is on record for `spans`. The answer is
463
- * kept per batch, as `takeAck` keeps the ack.
562
+ * it. Each retry and each half of a 413 split counts one; a call that the open circuit
563
+ * breaker or the quota pause stops sends none. Returns `null` when no call is on record for
564
+ * `spans`. The answer is kept per batch, as `takeAck` keeps the ack.
464
565
  */
465
566
  takeAttempts(spans) {
466
567
  const n = this._requests.get(spans) ?? null;
@@ -477,6 +578,12 @@ var HttpExporter = class {
477
578
  this._acks.delete(spans);
478
579
  return ack;
479
580
  }
581
+ /** Returns why the `export()` call for `spans` failed (W2-1c), and clears it. See `SpanExporter`. */
582
+ takeDropReason(spans) {
583
+ const reason = this._dropReasons.get(spans) ?? null;
584
+ this._dropReasons.delete(spans);
585
+ return reason;
586
+ }
480
587
  /** Returns the spans of `spans` that could not be serialized and were not sent, and clears them. */
481
588
  takeUnserializable(spans) {
482
589
  const bad = this._unserializable.get(spans) ?? [];
@@ -485,7 +592,7 @@ var HttpExporter = class {
485
592
  }
486
593
  /**
487
594
  * Cancel the `export()` call for `spans` (all calls when `spans` is not given): its
488
- * request, its retries and its backoff sleep. The batch processor calls it when
595
+ * request, its retries and its backoff wait. The batch processor calls it when
489
596
  * `shutdown()` abandons the batch, so that the call does not keep the process alive. The
490
597
  * cancelled call returns FAILURE, changes no counter (the circuit breaker included) and
491
598
  * logs no WARNING.
@@ -495,24 +602,39 @@ var HttpExporter = class {
495
602
  else for (const call of this._calls.values()) call.cancel.abort();
496
603
  }
497
604
  /**
498
- * Send one batch. The limits are the request timeout and `maxRetries`.
605
+ * End the retry waits of this exporter until its next `shutdown()`. The batch processor
606
+ * calls it when `shutdown()` starts. A call that waits for a retry makes one more attempt
607
+ * at once. An attempt that fails after this is not retried: the call fails with the drop
608
+ * reason `shutdown_residue`. So a drain against a dead endpoint costs one attempt for each
609
+ * batch, not the shutdown budget.
610
+ */
611
+ endRetries() {
612
+ this._noWait = true;
613
+ for (const wake of [...this._waits]) wake();
614
+ }
615
+ /**
616
+ * Send one batch, with the W2-1c retry policy (see the block above the class).
617
+ *
618
+ * Resolves SUCCESS when the backend accounted for the batch: `takeAck` has the counts (and
619
+ * the spans that a 413 split dropped, in `dropped`). Resolves FAILURE when no span was
620
+ * accounted for: `takeDropReason` then says why, and the batch processor drops the batch
621
+ * with that reason.
499
622
  *
500
623
  * ⚠ NO CALLER BUDGET. `flush(timeoutMs)` only stops WAITING at its deadline; it never
501
- * cuts a request that is already sent. A short flush budget that cut the request made a
502
- * healthy, slow server look like a failure: the span used a retry, the breaker moved, and
503
- * the server stored the same spans again. Only `abort()` stops the call, and only
624
+ * cuts a request that is already sent. Only `abort()` stops the call, and only
504
625
  * `shutdown()` calls it.
505
626
  */
506
627
  async export(spans) {
507
628
  if (spans.length === 0) return "success" /* SUCCESS */;
508
629
  const call = {
509
630
  cancel: new AbortController(),
510
- refusalAfterMs: null,
631
+ retryAfterS: null,
511
632
  misroute: null,
512
633
  noRetry: false,
513
634
  status: null,
514
635
  error: null,
515
- attempts: 0
636
+ attempts: 0,
637
+ ack: null
516
638
  };
517
639
  this._calls.set(spans, call);
518
640
  try {
@@ -549,83 +671,236 @@ var HttpExporter = class {
549
671
  );
550
672
  }
551
673
  if (parts.length === 0) return "success" /* SUCCESS */;
552
- const payload = `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
553
- const now = Date.now();
554
- let isHalfOpen = false;
555
- if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
556
- if (now < this._circuitOpenUntil) {
557
- return "failure" /* FAILURE */;
674
+ if (this._consecutiveFailures >= this._circuitBreakerThreshold && Date.now() < this._circuitOpenUntil) {
675
+ this._dropReasons.set(spans, "transport_refused");
676
+ return "failure" /* FAILURE */;
677
+ }
678
+ const out = { delivered: [], dropped: {}, failed: [], cancelled: false };
679
+ const start = performance.now();
680
+ await this._deliver(parts, start, call, out, true);
681
+ if (cancelled.aborted || out.cancelled) {
682
+ this._acks.delete(spans);
683
+ return "failure" /* FAILURE */;
684
+ }
685
+ return this._finish(spans, out, call.attempts);
686
+ }
687
+ /** The request body for `parts` (the serialized spans of one part of a batch). */
688
+ _payload(parts) {
689
+ return `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
690
+ }
691
+ /**
692
+ * Send one part of a batch with the retry policy, and record its outcome in `out`.
693
+ * `maySplit` is true only for the whole batch: a 413 splits it in two one time. A half
694
+ * that gets 413 again is dropped as `too_large`.
695
+ */
696
+ async _deliver(parts, start, call, out, maySplit) {
697
+ const n = parts.length;
698
+ if (!maySplit && this._quota.paused()) {
699
+ out.delivered.push({ n, ack: { accepted: 0, rejected: 0, quotaDropped: n } });
700
+ return;
701
+ }
702
+ const result = await this._sendWithRetries(this._payload(parts), n, start, call);
703
+ if (result.kind === "delivered") {
704
+ out.delivered.push({ n, ack: result.ack });
705
+ } else if (result.kind === "too_large") {
706
+ if (maySplit && n > 1) {
707
+ const mid = Math.ceil(n / 2);
708
+ debug(`Ingest answered HTTP 413 for ${n} span(s): sending two halves of ${mid} and ${n - mid}`);
709
+ await this._deliver(parts.slice(0, mid), start, call, out, false);
710
+ if (!out.cancelled) await this._deliver(parts.slice(mid), start, call, out, false);
711
+ } else {
712
+ out.dropped.too_large = (out.dropped.too_large ?? 0) + n;
558
713
  }
559
- isHalfOpen = true;
714
+ } else if (result.kind === "cancelled") {
715
+ out.cancelled = true;
716
+ } else {
717
+ out.failed.push({ n, reason: result.kind, answer: result.answer });
560
718
  }
561
- const maxAttempts = isHalfOpen ? 1 : this._maxRetries;
562
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
563
- call.refusalAfterMs = null;
719
+ }
720
+ /** Send one request body until it is delivered or the policy stops. */
721
+ async _sendWithRetries(body, spansSent, start, call) {
722
+ const cancelled = call.cancel.signal;
723
+ let retryNumber = 0;
724
+ let attemptsHere = 0;
725
+ for (; ; ) {
726
+ if (cancelled.aborted) return { kind: "cancelled" };
727
+ call.retryAfterS = null;
564
728
  call.misroute = null;
565
729
  call.noRetry = false;
566
730
  call.status = null;
567
731
  call.error = null;
732
+ call.ack = null;
568
733
  call.attempts++;
569
- const result = await this._sendRequest(payload, parts.length, call, spans);
570
- if (cancelled.aborted) {
571
- this._acks.delete(spans);
572
- return "failure" /* FAILURE */;
734
+ attemptsHere++;
735
+ this.attempts++;
736
+ const result = await this._sendRequest(body, spansSent, call);
737
+ if (cancelled.aborted) return { kind: "cancelled" };
738
+ if (result === "success" /* SUCCESS */) return { kind: "delivered", ack: call.ack };
739
+ if (call.status === TOO_LARGE_STATUS) return { kind: "too_large" };
740
+ const answer = {
741
+ status: call.status,
742
+ error: call.error,
743
+ misroute: call.misroute,
744
+ zeroAck: call.noRetry,
745
+ sent: spansSent
746
+ };
747
+ if (!this._retryable(call)) return { kind: "transport_refused", answer };
748
+ if (attemptsHere === RETRY_WARN_AFTER_ATTEMPTS) this._warnOutage(answer, true);
749
+ if (this._noWait) return { kind: "shutdown_residue", answer };
750
+ if (this._maxRetries !== void 0 && attemptsHere >= this._maxRetries) {
751
+ return { kind: "retry_exhausted", answer };
573
752
  }
574
- if (result === "success" /* SUCCESS */) {
575
- this._consecutiveFailures = 0;
576
- this._failureKind = null;
577
- return result;
578
- }
579
- if (call.misroute !== null || call.noRetry) break;
580
- if (call.status !== null && AUTH_REFUSAL_STATUSES.has(call.status)) break;
581
- if (attempt < maxAttempts - 1) {
582
- const refusal = call.refusalAfterMs;
583
- const delay = refusal === null ? 100 * Math.pow(2, attempt) : refusal === REFUSAL_NO_HINT ? REFUSAL_BACKOFF_BASE_MS * Math.pow(2, attempt) : refusal;
584
- await sleep(delay, cancelled);
585
- if (cancelled.aborted) return "failure" /* FAILURE */;
753
+ const remainingS = this._retryHorizonS - (performance.now() - start) / 1e3;
754
+ if (remainingS <= 0) return { kind: "retry_exhausted", answer };
755
+ const hint = call.retryAfterS;
756
+ const waitS = hint !== null && hint > 0 ? hint : backoffDelayS(retryNumber, jitter.fraction());
757
+ const delayS = Math.min(waitS, remainingS);
758
+ retryNumber++;
759
+ debug(
760
+ `Ingest attempt ${attemptsHere} failed (${answer.status ?? answer.error ?? "unknown"}); next attempt in ${delayS.toFixed(2)} s (${Math.round(remainingS)} s of the horizon left)`
761
+ );
762
+ await this._retryWait(delayS * 1e3, cancelled);
763
+ }
764
+ }
765
+ /** True when the answer of the last attempt of `call` can change on a retry. */
766
+ _retryable(call) {
767
+ if (call.misroute !== null || call.noRetry) return false;
768
+ if (call.status !== null) return RETRY_STATUSES.has(call.status);
769
+ if (this._endpointHasCredentials) return false;
770
+ return call.error !== null;
771
+ }
772
+ /**
773
+ * Wait `ms` before the next attempt. `abort()` and `endRetries()` end the wait at once. The
774
+ * timer does not keep the process alive: when nothing else runs, `beforeExit` starts the
775
+ * SDK's `shutdown()`, which ends the wait.
776
+ */
777
+ _retryWait(ms, signal) {
778
+ return new Promise((resolve) => {
779
+ if (signal.aborted || this._noWait || ms <= 0) return resolve();
780
+ const done = () => {
781
+ clearTimeout(timer);
782
+ signal.removeEventListener("abort", done);
783
+ this._waits.delete(done);
784
+ resolve();
785
+ };
786
+ const timer = setTimeout(done, Math.min(ms, MAX_TIMER_MS));
787
+ timer.unref?.();
788
+ signal.addEventListener("abort", done, { once: true });
789
+ this._waits.add(done);
790
+ });
791
+ }
792
+ /** Turn the outcome of the parts of one batch into the result of `export()`. */
793
+ _finish(spans, out, attempts) {
794
+ const reasons = new Set(out.failed.map((f) => f.reason));
795
+ if (out.delivered.length === 0 && Object.keys(out.dropped).length === 0 && reasons.size === 1) {
796
+ const reason = out.failed[0].reason;
797
+ const nFailed = out.failed.reduce((sum, f) => sum + f.n, 0);
798
+ this._dropReasons.set(spans, reason);
799
+ return this._fail(reason, out.failed[out.failed.length - 1].answer, nFailed, attempts);
800
+ }
801
+ let ack;
802
+ if (out.delivered.length === 1 && Object.keys(out.dropped).length === 0 && out.failed.length === 0) {
803
+ ack = out.delivered[0].ack;
804
+ } else {
805
+ let accepted = 0;
806
+ let rejected = 0;
807
+ let quotaDropped = 0;
808
+ for (const part of out.delivered) {
809
+ accepted += part.ack !== null ? part.ack.accepted : part.n;
810
+ rejected += part.ack !== null ? part.ack.rejected : 0;
811
+ quotaDropped += part.ack?.quotaDropped ?? 0;
586
812
  }
813
+ const dropped = { ...out.dropped };
814
+ for (const f of out.failed) dropped[f.reason] = (dropped[f.reason] ?? 0) + f.n;
815
+ ack = { accepted, rejected };
816
+ if (quotaDropped > 0) ack.quotaDropped = quotaDropped;
817
+ if (Object.keys(dropped).length > 0) ack.dropped = dropped;
587
818
  }
588
- if (call.refusalAfterMs !== null) {
589
- warn(
590
- `Risicare: the ingest endpoint refused this batch for capacity (HTTP 429/503) after ${call.attempts} attempt(s). This is a healthy server shedding load, not an outage. This export of the batch failed; the circuit breaker does not count it. Reduce batch size or export rate if it persists.`
591
- );
819
+ if (ack !== null) this._acks.set(spans, ack);
820
+ this._consecutiveFailures = 0;
821
+ this._failureKind = null;
822
+ const tooLarge = out.dropped.too_large ?? 0;
823
+ if (tooLarge > 0) this._warnTooLarge(tooLarge);
824
+ for (const f of out.failed) {
825
+ if (f.reason === "retry_exhausted") this._warnExhausted(f.n, attempts, f.answer);
826
+ }
827
+ return "success" /* SUCCESS */;
828
+ }
829
+ /** End a call in which no span was accounted for. Always returns FAILURE. */
830
+ _fail(reason, answer, nSpans, attempts) {
831
+ if (reason === "shutdown_residue") {
832
+ debug(`Export ended by shutdown after ${attempts} attempt(s)`);
592
833
  return "failure" /* FAILURE */;
593
834
  }
594
- const misroute = call.misroute;
595
- if (misroute !== null && Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
596
- this._lastMisrouteWarnAt = Date.now();
597
- warn(
598
- `Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
599
- );
835
+ if (reason === "retry_exhausted") {
836
+ if (answer.status === null || !CAPACITY_REFUSAL_STATUSES.has(answer.status)) this._stepBreaker();
837
+ this._warnOutage(answer, true);
838
+ this._warnExhausted(nSpans, attempts, answer);
839
+ return "failure" /* FAILURE */;
600
840
  }
841
+ this._stepBreaker();
842
+ this._warnOutage(answer, false);
843
+ return "failure" /* FAILURE */;
844
+ }
845
+ /** Count one failed call toward the circuit breaker. */
846
+ _stepBreaker() {
601
847
  this._consecutiveFailures++;
602
- if (misroute === null && !call.noRetry) {
603
- const status = call.status;
604
- const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : call.error !== null ? "none" : "unknown";
605
- let write = false;
606
- if (this._consecutiveFailures === 1 || this._failureKind === null) {
607
- if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
608
- this._lastFailureWarnAt = Date.now();
609
- write = true;
610
- }
611
- this._failureKind = kind;
612
- } else if (kind !== this._failureKind) {
613
- this._failureKind = kind;
614
- write = true;
615
- }
616
- if (write) this._warnOutage(kind, status, call.error);
617
- }
618
848
  if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
619
849
  this._circuitOpenUntil = Date.now() + this._circuitBreakerCooldownMs;
620
850
  warn(
621
851
  `HTTP exporter circuit breaker opened after ${this._consecutiveFailures} failures. Cooldown: ${this._circuitBreakerCooldownMs / 1e3}s`
622
852
  );
623
853
  }
624
- return "failure" /* FAILURE */;
625
854
  }
626
- /** The outage WARNING for one kind of failure. */
627
- _warnOutage(kind, status, error) {
855
+ /**
856
+ * The outage WARNING: at the first failure of an outage, and when its kind changes. At most
857
+ * one in the window for the first failure; a misroute has its own WARNING, at most one in
858
+ * the window. `retrying` is true for a retryable answer (written after
859
+ * `RETRY_WARN_AFTER_ATTEMPTS` failed attempts of one batch, or when its retries end before
860
+ * that), false for an answer that a retry cannot change (one attempt). The WARNING names the last HTTP status, or the error
861
+ * code when there was no answer. It never names the key or the response body, and names
862
+ * the API key as a cause only for a 401 or 403. An ack of 0 and 0 has its WARNING from
863
+ * `checkAck`.
864
+ */
865
+ _warnOutage(answer, retrying) {
866
+ if (answer.misroute !== null) {
867
+ if (Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
868
+ this._lastMisrouteWarnAt = Date.now();
869
+ warn(
870
+ `Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${answer.misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
871
+ );
872
+ }
873
+ return;
874
+ }
875
+ if (answer.zeroAck) return;
876
+ const status = answer.status;
877
+ const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : answer.error !== null ? "none" : "unknown";
878
+ let write = false;
879
+ if (this._failureKind === null || !retrying && this._consecutiveFailures === 1) {
880
+ if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
881
+ this._lastFailureWarnAt = Date.now();
882
+ write = true;
883
+ }
884
+ this._failureKind = kind;
885
+ } else if (kind !== this._failureKind) {
886
+ this._failureKind = kind;
887
+ write = true;
888
+ }
889
+ if (!write) return;
628
890
  const where = `Risicare: spans are NOT reaching ${this._safeEndpoint}`;
891
+ if (retrying) {
892
+ let limit = `${this._retryHorizonS} s from its first attempt (retryHorizonS)`;
893
+ if (this._maxRetries !== void 0) limit = `${this._maxRetries} attempt(s) (maxRetries) or ${limit}`;
894
+ const retryText = `The SDK sends each batch again with a growing wait, for up to ${limit}, and then drops it (reason=retry_exhausted).`;
895
+ if (kind === "http") {
896
+ warn(`${where}: it answered HTTP ${status}. ${retryText} Set debug: true to see each attempt.`);
897
+ } else {
898
+ warn(
899
+ `${where}: no answer (${answer.error}). The endpoint cannot be reached, or it did not answer in time. ${retryText} Set debug: true to see each attempt.`
900
+ );
901
+ }
902
+ return;
903
+ }
629
904
  const tail = "Tracing continues, but the SDK drops the spans that it cannot send.";
630
905
  if (kind === "auth") {
631
906
  warn(
@@ -633,15 +908,11 @@ var HttpExporter = class {
633
908
  );
634
909
  } else if (kind === "http") {
635
910
  warn(
636
- `${where}: it answered HTTP ${status}. The server did not take the batch. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
911
+ `${where}: it answered HTTP ${status}. The server did not take the batch, and a retry cannot change that answer. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
637
912
  );
638
913
  } else if (kind === "none" && this._endpointHasCredentials) {
639
914
  warn(
640
- `${where}: no answer (${error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
641
- );
642
- } else if (kind === "none") {
643
- warn(
644
- `${where}: no answer (${error}). The endpoint cannot be reached, or it did not answer in time. Check the endpoint and the network. ${tail} Set debug: true to see each attempt.`
915
+ `${where}: no answer (${answer.error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
645
916
  );
646
917
  } else {
647
918
  warn(
@@ -649,33 +920,38 @@ var HttpExporter = class {
649
920
  );
650
921
  }
651
922
  }
923
+ /** The WARNING for spans dropped as `retry_exhausted`. At most one in the window. */
924
+ _warnExhausted(nSpans, attempts, answer) {
925
+ if (Date.now() - this._lastExhaustedWarnAt <= FAILURE_WARN_WINDOW_MS) return;
926
+ this._lastExhaustedWarnAt = Date.now();
927
+ const last = answer.status !== null ? `HTTP ${answer.status}` : `no answer (${answer.error})`;
928
+ const capacity = answer.status !== null && CAPACITY_REFUSAL_STATUSES.has(answer.status) ? " (over capacity)" : "";
929
+ const limit = this._maxRetries !== void 0 && attempts >= this._maxRetries ? `the maxRetries limit of ${this._maxRetries} attempt(s)` : `the retry horizon of ${this._retryHorizonS} s (retryHorizonS)`;
930
+ warn(
931
+ `Risicare: ${nSpans} span(s) DISCARDED (reason=retry_exhausted): ${this._safeEndpoint} did not take them in ${attempts} attempt(s), and ${limit} is used up. The last answer: ${last}${capacity}. A gateway outage that is longer than the retry horizon loses spans. To wait longer, raise retryHorizonS.`
932
+ );
933
+ }
934
+ /** The WARNING for spans dropped as `too_large`. One time for each exporter. */
935
+ _warnTooLarge(nSpans) {
936
+ if (this._tooLargeWarned) return;
937
+ this._tooLargeWarned = true;
938
+ warn(
939
+ `Risicare: ${nSpans} span(s) DISCARDED (reason=too_large): ${this._safeEndpoint} answered HTTP 413 (the request is too large) for a batch, and again for a half of it after one split. The SDK does not send these spans again. Make the span attributes smaller. This WARNING is written one time; droppedSpansByReason.too_large has the count.`
940
+ );
941
+ }
652
942
  /** Record a MISROUTE for the retry loop and the breaker. Always returns FAILURE. */
653
943
  _noteMisroute(call, answer) {
654
944
  call.misroute = this._scrub(answer, 300);
655
945
  debug(`Ingest misroute: ${call.misroute}`);
656
946
  return "failure" /* FAILURE */;
657
947
  }
658
- /**
659
- * Record a CAPACITY REFUSAL for the retry loop. Returns true if it was one.
660
- *
661
- * `Retry-After` is honoured only in its delta-seconds form (RFC 9110 §10.2.3 also permits an
662
- * HTTP-date); anything else falls back to our own backoff rather than guessing at a date
663
- * parse.
664
- */
665
- _noteRefusal(call, status, retryAfterRaw) {
666
- if (!CAPACITY_REFUSAL_STATUSES.has(status)) return false;
667
- let delay = REFUSAL_NO_HINT;
668
- if (retryAfterRaw) {
669
- const secs = Number(retryAfterRaw);
670
- if (Number.isFinite(secs) && secs >= 0) {
671
- delay = Math.min(secs * 1e3, RETRY_AFTER_CEILING_MS);
672
- }
673
- }
674
- call.refusalAfterMs = delay;
675
- debug(`Ingest refused for capacity: HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
676
- return true;
948
+ /** Record the `Retry-After` wait of a retryable answer for the retry loop. */
949
+ _noteRetryAfter(call, status, retryAfterRaw) {
950
+ if (!RETRY_STATUSES.has(status)) return;
951
+ call.retryAfterS = parseRetryAfterS(retryAfterRaw);
952
+ debug(`Ingest answered HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
677
953
  }
678
- async _sendRequest(body, spansSent, call, spans) {
954
+ async _sendRequest(body, spansSent, call) {
679
955
  const cancelled = call.cancel.signal;
680
956
  const timeoutMs = this._timeoutMs;
681
957
  const url = `${this._endpoint}/v1/spans`;
@@ -743,14 +1019,15 @@ var HttpExporter = class {
743
1019
  } else if (ack.result === "success" /* SUCCESS */) {
744
1020
  this._quota.accepted(`the ingest endpoint ${this._safeEndpoint}`);
745
1021
  }
746
- if (ack.result === "success" /* SUCCESS */) this._acks.set(spans, ack.counts);
1022
+ if (ack.result === "success" /* SUCCESS */) call.ack = ack.counts;
747
1023
  if (ack.final) call.noRetry = true;
748
1024
  return ack.result;
749
- } else if (response.status === 408 || response.status === 504) {
1025
+ }
1026
+ this._noteRetryAfter(call, response.status, response.headers?.get?.("Retry-After") ?? null);
1027
+ if (response.status === 408 || response.status === 504) {
750
1028
  debug(`HTTP export timeout: ${response.status}`);
751
1029
  return "timeout" /* TIMEOUT */;
752
1030
  }
753
- this._noteRefusal(call, response.status, response.headers.get("Retry-After"));
754
1031
  debug(`HTTP export failed: ${response.status}`);
755
1032
  return "failure" /* FAILURE */;
756
1033
  } catch (e) {
@@ -764,20 +1041,9 @@ var HttpExporter = class {
764
1041
  }
765
1042
  }
766
1043
  shutdown() {
1044
+ this._noWait = false;
767
1045
  }
768
1046
  };
769
- function sleep(ms, signal) {
770
- return new Promise((resolve) => {
771
- if (signal?.aborted) return resolve();
772
- const done = () => {
773
- clearTimeout(timer);
774
- signal?.removeEventListener("abort", done);
775
- resolve();
776
- };
777
- const timer = setTimeout(done, ms);
778
- signal?.addEventListener("abort", done, { once: true });
779
- });
780
- }
781
1047
  function asCount(v) {
782
1048
  return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : void 0;
783
1049
  }