risicare 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +76 -5
  2. package/README.md +157 -39
  3. package/dist/frameworks/instructor.cjs +338 -117
  4. package/dist/frameworks/instructor.cjs.map +1 -1
  5. package/dist/frameworks/instructor.js +338 -117
  6. package/dist/frameworks/instructor.js.map +1 -1
  7. package/dist/frameworks/langchain.cjs +351 -130
  8. package/dist/frameworks/langchain.cjs.map +1 -1
  9. package/dist/frameworks/langchain.js +351 -130
  10. package/dist/frameworks/langchain.js.map +1 -1
  11. package/dist/frameworks/langgraph.cjs +351 -130
  12. package/dist/frameworks/langgraph.cjs.map +1 -1
  13. package/dist/frameworks/langgraph.js +351 -130
  14. package/dist/frameworks/langgraph.js.map +1 -1
  15. package/dist/frameworks/llamaindex.cjs +351 -130
  16. package/dist/frameworks/llamaindex.cjs.map +1 -1
  17. package/dist/frameworks/llamaindex.js +351 -130
  18. package/dist/frameworks/llamaindex.js.map +1 -1
  19. package/dist/index.cjs +808 -226
  20. package/dist/index.cjs.map +1 -1
  21. package/dist/index.d.cts +137 -21
  22. package/dist/index.d.ts +137 -21
  23. package/dist/index.js +804 -226
  24. package/dist/index.js.map +1 -1
  25. package/dist/providers/anthropic/index.cjs +351 -130
  26. package/dist/providers/anthropic/index.cjs.map +1 -1
  27. package/dist/providers/anthropic/index.js +351 -130
  28. package/dist/providers/anthropic/index.js.map +1 -1
  29. package/dist/providers/bedrock/index.cjs +346 -125
  30. package/dist/providers/bedrock/index.cjs.map +1 -1
  31. package/dist/providers/bedrock/index.js +346 -125
  32. package/dist/providers/bedrock/index.js.map +1 -1
  33. package/dist/providers/cerebras/index.cjs +351 -130
  34. package/dist/providers/cerebras/index.cjs.map +1 -1
  35. package/dist/providers/cerebras/index.js +351 -130
  36. package/dist/providers/cerebras/index.js.map +1 -1
  37. package/dist/providers/cohere/index.cjs +351 -130
  38. package/dist/providers/cohere/index.cjs.map +1 -1
  39. package/dist/providers/cohere/index.js +351 -130
  40. package/dist/providers/cohere/index.js.map +1 -1
  41. package/dist/providers/google/index.cjs +351 -130
  42. package/dist/providers/google/index.cjs.map +1 -1
  43. package/dist/providers/google/index.js +351 -130
  44. package/dist/providers/google/index.js.map +1 -1
  45. package/dist/providers/groq/index.cjs +351 -130
  46. package/dist/providers/groq/index.cjs.map +1 -1
  47. package/dist/providers/groq/index.js +351 -130
  48. package/dist/providers/groq/index.js.map +1 -1
  49. package/dist/providers/huggingface/index.cjs +351 -130
  50. package/dist/providers/huggingface/index.cjs.map +1 -1
  51. package/dist/providers/huggingface/index.js +351 -130
  52. package/dist/providers/huggingface/index.js.map +1 -1
  53. package/dist/providers/mistral/index.cjs +351 -130
  54. package/dist/providers/mistral/index.cjs.map +1 -1
  55. package/dist/providers/mistral/index.js +351 -130
  56. package/dist/providers/mistral/index.js.map +1 -1
  57. package/dist/providers/ollama/index.cjs +351 -130
  58. package/dist/providers/ollama/index.cjs.map +1 -1
  59. package/dist/providers/ollama/index.js +351 -130
  60. package/dist/providers/ollama/index.js.map +1 -1
  61. package/dist/providers/openai/index.cjs +351 -130
  62. package/dist/providers/openai/index.cjs.map +1 -1
  63. package/dist/providers/openai/index.js +351 -130
  64. package/dist/providers/openai/index.js.map +1 -1
  65. package/dist/providers/together/index.cjs +351 -130
  66. package/dist/providers/together/index.cjs.map +1 -1
  67. package/dist/providers/together/index.js +351 -130
  68. package/dist/providers/together/index.js.map +1 -1
  69. package/dist/providers/vercel-ai/index.cjs +351 -130
  70. package/dist/providers/vercel-ai/index.cjs.map +1 -1
  71. package/dist/providers/vercel-ai/index.js +351 -130
  72. package/dist/providers/vercel-ai/index.js.map +1 -1
  73. package/package.json +1 -1
@@ -37,6 +37,35 @@ module.exports = __toCommonJS(llamaindex_exports);
37
37
  // src/client.ts
38
38
  var import_node_crypto3 = require("crypto");
39
39
 
40
+ // src/exporters/retry.ts
41
+ var RETRY_STATUSES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
42
+ var TOO_LARGE_STATUS = 413;
43
+ var RETRY_BACKOFF_BASE_S = 1;
44
+ var RETRY_BACKOFF_CAP_S = 30;
45
+ var DEFAULT_RETRY_HORIZON_S = 300;
46
+ var DEFAULT_SHUTDOWN_TIMEOUT_S = 10;
47
+ var RETRY_WARN_AFTER_ATTEMPTS = 3;
48
+ var jitter = { fraction: () => Math.random() };
49
+ function backoffDelayS(retryNumber, fraction) {
50
+ const exponent = Math.min(Math.max(retryNumber, 0), 62);
51
+ const ceiling = Math.min(RETRY_BACKOFF_CAP_S, RETRY_BACKOFF_BASE_S * 2 ** exponent);
52
+ return Math.min(Math.max(fraction, 0), 1) * ceiling;
53
+ }
54
+ var SECONDS = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
55
+ function parseRetryAfterS(value, nowMs = Date.now()) {
56
+ if (value === null || value === void 0) return null;
57
+ const text = String(value).trim();
58
+ if (text === "") return null;
59
+ if (SECONDS.test(text)) {
60
+ const seconds = Number(text);
61
+ return Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
62
+ }
63
+ if (!/[A-Za-z]/.test(text)) return null;
64
+ const when = Date.parse(text);
65
+ if (!Number.isFinite(when)) return null;
66
+ return Math.max(0, (when - nowMs) / 1e3);
67
+ }
68
+
40
69
  // src/globals.ts
41
70
  var import_node_async_hooks = require("async_hooks");
42
71
  var G = globalThis;
@@ -183,10 +212,24 @@ var VALUE_RULE_PREFIXES = new Set(
183
212
  );
184
213
 
185
214
  // src/exporters/batch.ts
215
+ var SHUTDOWN_BUDGET_MS = DEFAULT_SHUTDOWN_TIMEOUT_S * 1e3;
186
216
  var MAX_TIMEOUT_MS = 2 ** 31 - 1;
187
217
  var MAX_BATCH_BYTES = 8 * 1024 * 1024;
188
218
  var utf8ByteLength = typeof Buffer !== "undefined" && typeof Buffer.byteLength === "function" ? (s) => Buffer.byteLength(s, "utf8") : (s) => s.length * 3;
189
219
 
220
+ // src/context/storage.ts
221
+ function storage() {
222
+ return getContextStorage();
223
+ }
224
+ function getContext() {
225
+ return storage().getStore() ?? {};
226
+ }
227
+ function runWithContext(overrides, fn) {
228
+ const parent = getContext();
229
+ const merged = { ...parent, ...overrides };
230
+ return storage().run(merged, fn);
231
+ }
232
+
190
233
  // src/ids.ts
191
234
  var import_node_crypto = require("crypto");
192
235
 
@@ -317,21 +360,8 @@ var SUPPLEMENTAL_RULES = [
317
360
  { re: new RegExp("json|syntaxerror|unexpected token|decodeerror", "u"), code: "PERCEPTION.PARSING.JSON_INVALID" }
318
361
  ];
319
362
 
320
- // src/context/storage.ts
321
- function storage() {
322
- return getContextStorage();
323
- }
324
- function getContext() {
325
- return storage().getStore() ?? {};
326
- }
327
- function runWithContext(overrides, fn) {
328
- const parent = getContext();
329
- const merged = { ...parent, ...overrides };
330
- return storage().run(merged, fn);
331
- }
332
-
333
363
  // src/version.ts
334
- var SDK_VERSION = "0.10.0";
364
+ var SDK_VERSION = "0.11.0";
335
365
 
336
366
  // src/exporters/quota.ts
337
367
  var QUOTA_REASON = "monthly_quota_exceeded";
@@ -439,12 +469,18 @@ var QuotaPause = class {
439
469
 
440
470
  // src/exporters/http.ts
441
471
  var CAPACITY_REFUSAL_STATUSES = /* @__PURE__ */ new Set([429, 503]);
442
- var RETRY_AFTER_CEILING_MS = 1e4;
443
- var REFUSAL_BACKOFF_BASE_MS = 1e3;
444
- var REFUSAL_NO_HINT = -1;
445
472
  var MISROUTE_WARN_WINDOW_MS = 3e5;
446
473
  var AUTH_REFUSAL_STATUSES = /* @__PURE__ */ new Set([401, 403]);
447
474
  var FAILURE_WARN_WINDOW_MS = 3e5;
475
+ var MAX_TIMER_MS = 2 ** 31 - 1;
476
+ function validHorizonS(value) {
477
+ if (value === void 0) return DEFAULT_RETRY_HORIZON_S;
478
+ if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
479
+ warn(
480
+ `Risicare: retryHorizonS=${String(value)} is not a finite number of 0 or more; using the default, ${DEFAULT_RETRY_HORIZON_S} s.`
481
+ );
482
+ return DEFAULT_RETRY_HORIZON_S;
483
+ }
448
484
  var HttpExporter = class {
449
485
  name = "http";
450
486
  _endpoint;
@@ -454,20 +490,28 @@ var HttpExporter = class {
454
490
  _timeoutMs;
455
491
  _maxRetries;
456
492
  _compress;
493
+ _retryHorizonS;
494
+ /** Total HTTP requests that this exporter sent (each retry and each half of a split counts). */
495
+ attempts = 0;
457
496
  // Circuit breaker (CON-201 / B-16). Threshold and cooldown sized for
458
- // a "30-second prod restart" scenario: with default 1s batch flush a
459
- // 5-failure threshold opens the circuit ~5s into an outage and the 60s
460
- // cooldown gives the gateway time to come back before the half-open
461
- // probe fires. Lower numbers (was 3@30s) tripped on transient blips.
497
+ // a "30-second prod restart" scenario. Since W2-1c only answers that a retry
498
+ // cannot change move it (see the block above).
462
499
  _consecutiveFailures = 0;
463
500
  _lastMisrouteWarnAt = -Infinity;
464
501
  _lastFailureWarnAt = -Infinity;
502
+ _lastExhaustedWarnAt = -Infinity;
503
+ /** "log once" (W2-1c): the WARNING for a `too_large` drop is written one time. */
504
+ _tooLargeWarned = false;
465
505
  /**
466
506
  * The kind of failure that the last outage WARNING named ('auth', 'http', 'none',
467
507
  * 'unknown'). `null` after a success. A new kind in the same outage writes one more
468
508
  * WARNING, also inside the window.
469
509
  */
470
510
  _failureKind = null;
511
+ /** `endRetries()` was called, and `shutdown()` has not run since. */
512
+ _noWait = false;
513
+ /** The wake-ups of the backoff waits in progress. `endRetries()` calls each one. */
514
+ _waits = /* @__PURE__ */ new Set();
471
515
  /** The endpoint as a log line may show it: no userinfo, no query (see `safeUrl`). */
472
516
  _safeEndpoint;
473
517
  /** The endpoint holds userinfo. `fetch` refuses such a URL. */
@@ -478,6 +522,8 @@ var HttpExporter = class {
478
522
  _acks = /* @__PURE__ */ new WeakMap();
479
523
  /** The requests that each finished call sent, by its batch. Read once by `takeAttempts(spans)`. */
480
524
  _requests = /* @__PURE__ */ new WeakMap();
525
+ /** The drop reason of each failed call, by its batch. Read once by `takeDropReason(spans)`. */
526
+ _dropReasons = /* @__PURE__ */ new WeakMap();
481
527
  /** The batch processor dropped span(s) from a full queue in the quota pause (S-1). */
482
528
  noteQuotaDrop(count = 1) {
483
529
  this._quota.addDrops(count);
@@ -502,8 +548,16 @@ var HttpExporter = class {
502
548
  this._projectId = options.projectId;
503
549
  this._environment = options.environment;
504
550
  this._timeoutMs = options.timeoutMs ?? 5e3;
505
- this._maxRetries = options.maxRetries ?? 3;
551
+ let maxRetries = options.maxRetries;
552
+ if (maxRetries !== void 0 && !(Number.isInteger(maxRetries) && maxRetries >= 1)) {
553
+ warn(
554
+ `Risicare: maxRetries=${String(maxRetries)} is not a whole number of 1 or more; the exporter uses no count limit (the retry horizon is the limit).`
555
+ );
556
+ maxRetries = void 0;
557
+ }
558
+ this._maxRetries = maxRetries;
506
559
  this._compress = options.compress ?? false;
560
+ this._retryHorizonS = validHorizonS(options.retryHorizonS);
507
561
  this._safeEndpoint = safeUrl(this._endpoint);
508
562
  this._endpointHasCredentials = hasCredentials(this._endpoint);
509
563
  }
@@ -525,16 +579,18 @@ var HttpExporter = class {
525
579
  [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
526
580
  return `HttpExporter { endpoint: '${this._safeEndpoint}' }`;
527
581
  }
528
- /** The most requests that one `export()` call sends (`maxRetries`). */
582
+ /**
583
+ * The most attempts for one request body (`maxRetries`). `undefined` when no count is set:
584
+ * the retry horizon is the limit.
585
+ */
529
586
  get attemptsPerExport() {
530
587
  return this._maxRetries;
531
588
  }
532
589
  /**
533
590
  * Returns the number of HTTP requests that the `export()` call for `spans` sent, and clears
534
- * it. This is the number that was sent, not the most that the call may send
535
- * (`attemptsPerExport`): a 401 or 403 stops the loop after one request, and an open circuit
536
- * breaker sends none. Returns `null` when no call is on record for `spans`. The answer is
537
- * kept per batch, as `takeAck` keeps the ack.
591
+ * it. Each retry and each half of a 413 split counts one; a call that the open circuit
592
+ * breaker or the quota pause stops sends none. Returns `null` when no call is on record for
593
+ * `spans`. The answer is kept per batch, as `takeAck` keeps the ack.
538
594
  */
539
595
  takeAttempts(spans) {
540
596
  const n = this._requests.get(spans) ?? null;
@@ -551,6 +607,12 @@ var HttpExporter = class {
551
607
  this._acks.delete(spans);
552
608
  return ack;
553
609
  }
610
+ /** Returns why the `export()` call for `spans` failed (W2-1c), and clears it. See `SpanExporter`. */
611
+ takeDropReason(spans) {
612
+ const reason = this._dropReasons.get(spans) ?? null;
613
+ this._dropReasons.delete(spans);
614
+ return reason;
615
+ }
554
616
  /** Returns the spans of `spans` that could not be serialized and were not sent, and clears them. */
555
617
  takeUnserializable(spans) {
556
618
  const bad = this._unserializable.get(spans) ?? [];
@@ -559,7 +621,7 @@ var HttpExporter = class {
559
621
  }
560
622
  /**
561
623
  * Cancel the `export()` call for `spans` (all calls when `spans` is not given): its
562
- * request, its retries and its backoff sleep. The batch processor calls it when
624
+ * request, its retries and its backoff wait. The batch processor calls it when
563
625
  * `shutdown()` abandons the batch, so that the call does not keep the process alive. The
564
626
  * cancelled call returns FAILURE, changes no counter (the circuit breaker included) and
565
627
  * logs no WARNING.
@@ -569,24 +631,39 @@ var HttpExporter = class {
569
631
  else for (const call of this._calls.values()) call.cancel.abort();
570
632
  }
571
633
  /**
572
- * Send one batch. The limits are the request timeout and `maxRetries`.
634
+ * End the retry waits of this exporter until its next `shutdown()`. The batch processor
635
+ * calls it when `shutdown()` starts. A call that waits for a retry makes one more attempt
636
+ * at once. An attempt that fails after this is not retried: the call fails with the drop
637
+ * reason `shutdown_residue`. So a drain against a dead endpoint costs one attempt for each
638
+ * batch, not the shutdown budget.
639
+ */
640
+ endRetries() {
641
+ this._noWait = true;
642
+ for (const wake of [...this._waits]) wake();
643
+ }
644
+ /**
645
+ * Send one batch, with the W2-1c retry policy (see the block above the class).
646
+ *
647
+ * Resolves SUCCESS when the backend accounted for the batch: `takeAck` has the counts (and
648
+ * the spans that a 413 split dropped, in `dropped`). Resolves FAILURE when no span was
649
+ * accounted for: `takeDropReason` then says why, and the batch processor drops the batch
650
+ * with that reason.
573
651
  *
574
652
  * ⚠ NO CALLER BUDGET. `flush(timeoutMs)` only stops WAITING at its deadline; it never
575
- * cuts a request that is already sent. A short flush budget that cut the request made a
576
- * healthy, slow server look like a failure: the span used a retry, the breaker moved, and
577
- * the server stored the same spans again. Only `abort()` stops the call, and only
653
+ * cuts a request that is already sent. Only `abort()` stops the call, and only
578
654
  * `shutdown()` calls it.
579
655
  */
580
656
  async export(spans) {
581
657
  if (spans.length === 0) return "success" /* SUCCESS */;
582
658
  const call = {
583
659
  cancel: new AbortController(),
584
- refusalAfterMs: null,
660
+ retryAfterS: null,
585
661
  misroute: null,
586
662
  noRetry: false,
587
663
  status: null,
588
664
  error: null,
589
- attempts: 0
665
+ attempts: 0,
666
+ ack: null
590
667
  };
591
668
  this._calls.set(spans, call);
592
669
  try {
@@ -623,83 +700,236 @@ var HttpExporter = class {
623
700
  );
624
701
  }
625
702
  if (parts.length === 0) return "success" /* SUCCESS */;
626
- const payload = `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
627
- const now = Date.now();
628
- let isHalfOpen = false;
629
- if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
630
- if (now < this._circuitOpenUntil) {
631
- return "failure" /* FAILURE */;
703
+ if (this._consecutiveFailures >= this._circuitBreakerThreshold && Date.now() < this._circuitOpenUntil) {
704
+ this._dropReasons.set(spans, "transport_refused");
705
+ return "failure" /* FAILURE */;
706
+ }
707
+ const out = { delivered: [], dropped: {}, failed: [], cancelled: false };
708
+ const start = performance.now();
709
+ await this._deliver(parts, start, call, out, true);
710
+ if (cancelled.aborted || out.cancelled) {
711
+ this._acks.delete(spans);
712
+ return "failure" /* FAILURE */;
713
+ }
714
+ return this._finish(spans, out, call.attempts);
715
+ }
716
+ /** The request body for `parts` (the serialized spans of one part of a batch). */
717
+ _payload(parts) {
718
+ return `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
719
+ }
720
+ /**
721
+ * Send one part of a batch with the retry policy, and record its outcome in `out`.
722
+ * `maySplit` is true only for the whole batch: a 413 splits it in two one time. A half
723
+ * that gets 413 again is dropped as `too_large`.
724
+ */
725
+ async _deliver(parts, start, call, out, maySplit) {
726
+ const n = parts.length;
727
+ if (!maySplit && this._quota.paused()) {
728
+ out.delivered.push({ n, ack: { accepted: 0, rejected: 0, quotaDropped: n } });
729
+ return;
730
+ }
731
+ const result = await this._sendWithRetries(this._payload(parts), n, start, call);
732
+ if (result.kind === "delivered") {
733
+ out.delivered.push({ n, ack: result.ack });
734
+ } else if (result.kind === "too_large") {
735
+ if (maySplit && n > 1) {
736
+ const mid = Math.ceil(n / 2);
737
+ debug(`Ingest answered HTTP 413 for ${n} span(s): sending two halves of ${mid} and ${n - mid}`);
738
+ await this._deliver(parts.slice(0, mid), start, call, out, false);
739
+ if (!out.cancelled) await this._deliver(parts.slice(mid), start, call, out, false);
740
+ } else {
741
+ out.dropped.too_large = (out.dropped.too_large ?? 0) + n;
632
742
  }
633
- isHalfOpen = true;
743
+ } else if (result.kind === "cancelled") {
744
+ out.cancelled = true;
745
+ } else {
746
+ out.failed.push({ n, reason: result.kind, answer: result.answer });
634
747
  }
635
- const maxAttempts = isHalfOpen ? 1 : this._maxRetries;
636
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
637
- call.refusalAfterMs = null;
748
+ }
749
+ /** Send one request body until it is delivered or the policy stops. */
750
+ async _sendWithRetries(body, spansSent, start, call) {
751
+ const cancelled = call.cancel.signal;
752
+ let retryNumber = 0;
753
+ let attemptsHere = 0;
754
+ for (; ; ) {
755
+ if (cancelled.aborted) return { kind: "cancelled" };
756
+ call.retryAfterS = null;
638
757
  call.misroute = null;
639
758
  call.noRetry = false;
640
759
  call.status = null;
641
760
  call.error = null;
761
+ call.ack = null;
642
762
  call.attempts++;
643
- const result = await this._sendRequest(payload, parts.length, call, spans);
644
- if (cancelled.aborted) {
645
- this._acks.delete(spans);
646
- return "failure" /* FAILURE */;
763
+ attemptsHere++;
764
+ this.attempts++;
765
+ const result = await this._sendRequest(body, spansSent, call);
766
+ if (cancelled.aborted) return { kind: "cancelled" };
767
+ if (result === "success" /* SUCCESS */) return { kind: "delivered", ack: call.ack };
768
+ if (call.status === TOO_LARGE_STATUS) return { kind: "too_large" };
769
+ const answer = {
770
+ status: call.status,
771
+ error: call.error,
772
+ misroute: call.misroute,
773
+ zeroAck: call.noRetry,
774
+ sent: spansSent
775
+ };
776
+ if (!this._retryable(call)) return { kind: "transport_refused", answer };
777
+ if (attemptsHere === RETRY_WARN_AFTER_ATTEMPTS) this._warnOutage(answer, true);
778
+ if (this._noWait) return { kind: "shutdown_residue", answer };
779
+ if (this._maxRetries !== void 0 && attemptsHere >= this._maxRetries) {
780
+ return { kind: "retry_exhausted", answer };
647
781
  }
648
- if (result === "success" /* SUCCESS */) {
649
- this._consecutiveFailures = 0;
650
- this._failureKind = null;
651
- return result;
652
- }
653
- if (call.misroute !== null || call.noRetry) break;
654
- if (call.status !== null && AUTH_REFUSAL_STATUSES.has(call.status)) break;
655
- if (attempt < maxAttempts - 1) {
656
- const refusal = call.refusalAfterMs;
657
- const delay = refusal === null ? 100 * Math.pow(2, attempt) : refusal === REFUSAL_NO_HINT ? REFUSAL_BACKOFF_BASE_MS * Math.pow(2, attempt) : refusal;
658
- await sleep(delay, cancelled);
659
- if (cancelled.aborted) return "failure" /* FAILURE */;
782
+ const remainingS = this._retryHorizonS - (performance.now() - start) / 1e3;
783
+ if (remainingS <= 0) return { kind: "retry_exhausted", answer };
784
+ const hint = call.retryAfterS;
785
+ const waitS = hint !== null && hint > 0 ? hint : backoffDelayS(retryNumber, jitter.fraction());
786
+ const delayS = Math.min(waitS, remainingS);
787
+ retryNumber++;
788
+ debug(
789
+ `Ingest attempt ${attemptsHere} failed (${answer.status ?? answer.error ?? "unknown"}); next attempt in ${delayS.toFixed(2)} s (${Math.round(remainingS)} s of the horizon left)`
790
+ );
791
+ await this._retryWait(delayS * 1e3, cancelled);
792
+ }
793
+ }
794
+ /** True when the answer of the last attempt of `call` can change on a retry. */
795
+ _retryable(call) {
796
+ if (call.misroute !== null || call.noRetry) return false;
797
+ if (call.status !== null) return RETRY_STATUSES.has(call.status);
798
+ if (this._endpointHasCredentials) return false;
799
+ return call.error !== null;
800
+ }
801
+ /**
802
+ * Wait `ms` before the next attempt. `abort()` and `endRetries()` end the wait at once. The
803
+ * timer does not keep the process alive: when nothing else runs, `beforeExit` starts the
804
+ * SDK's `shutdown()`, which ends the wait.
805
+ */
806
+ _retryWait(ms, signal) {
807
+ return new Promise((resolve) => {
808
+ if (signal.aborted || this._noWait || ms <= 0) return resolve();
809
+ const done = () => {
810
+ clearTimeout(timer);
811
+ signal.removeEventListener("abort", done);
812
+ this._waits.delete(done);
813
+ resolve();
814
+ };
815
+ const timer = setTimeout(done, Math.min(ms, MAX_TIMER_MS));
816
+ timer.unref?.();
817
+ signal.addEventListener("abort", done, { once: true });
818
+ this._waits.add(done);
819
+ });
820
+ }
821
+ /** Turn the outcome of the parts of one batch into the result of `export()`. */
822
+ _finish(spans, out, attempts) {
823
+ const reasons = new Set(out.failed.map((f) => f.reason));
824
+ if (out.delivered.length === 0 && Object.keys(out.dropped).length === 0 && reasons.size === 1) {
825
+ const reason = out.failed[0].reason;
826
+ const nFailed = out.failed.reduce((sum, f) => sum + f.n, 0);
827
+ this._dropReasons.set(spans, reason);
828
+ return this._fail(reason, out.failed[out.failed.length - 1].answer, nFailed, attempts);
829
+ }
830
+ let ack;
831
+ if (out.delivered.length === 1 && Object.keys(out.dropped).length === 0 && out.failed.length === 0) {
832
+ ack = out.delivered[0].ack;
833
+ } else {
834
+ let accepted = 0;
835
+ let rejected = 0;
836
+ let quotaDropped = 0;
837
+ for (const part of out.delivered) {
838
+ accepted += part.ack !== null ? part.ack.accepted : part.n;
839
+ rejected += part.ack !== null ? part.ack.rejected : 0;
840
+ quotaDropped += part.ack?.quotaDropped ?? 0;
660
841
  }
842
+ const dropped = { ...out.dropped };
843
+ for (const f of out.failed) dropped[f.reason] = (dropped[f.reason] ?? 0) + f.n;
844
+ ack = { accepted, rejected };
845
+ if (quotaDropped > 0) ack.quotaDropped = quotaDropped;
846
+ if (Object.keys(dropped).length > 0) ack.dropped = dropped;
661
847
  }
662
- if (call.refusalAfterMs !== null) {
663
- warn(
664
- `Risicare: the ingest endpoint refused this batch for capacity (HTTP 429/503) after ${call.attempts} attempt(s). This is a healthy server shedding load, not an outage. This export of the batch failed; the circuit breaker does not count it. Reduce batch size or export rate if it persists.`
665
- );
848
+ if (ack !== null) this._acks.set(spans, ack);
849
+ this._consecutiveFailures = 0;
850
+ this._failureKind = null;
851
+ const tooLarge = out.dropped.too_large ?? 0;
852
+ if (tooLarge > 0) this._warnTooLarge(tooLarge);
853
+ for (const f of out.failed) {
854
+ if (f.reason === "retry_exhausted") this._warnExhausted(f.n, attempts, f.answer);
855
+ }
856
+ return "success" /* SUCCESS */;
857
+ }
858
+ /** End a call in which no span was accounted for. Always returns FAILURE. */
859
+ _fail(reason, answer, nSpans, attempts) {
860
+ if (reason === "shutdown_residue") {
861
+ debug(`Export ended by shutdown after ${attempts} attempt(s)`);
666
862
  return "failure" /* FAILURE */;
667
863
  }
668
- const misroute = call.misroute;
669
- if (misroute !== null && Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
670
- this._lastMisrouteWarnAt = Date.now();
671
- warn(
672
- `Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
673
- );
864
+ if (reason === "retry_exhausted") {
865
+ if (answer.status === null || !CAPACITY_REFUSAL_STATUSES.has(answer.status)) this._stepBreaker();
866
+ this._warnOutage(answer, true);
867
+ this._warnExhausted(nSpans, attempts, answer);
868
+ return "failure" /* FAILURE */;
674
869
  }
870
+ this._stepBreaker();
871
+ this._warnOutage(answer, false);
872
+ return "failure" /* FAILURE */;
873
+ }
874
+ /** Count one failed call toward the circuit breaker. */
875
+ _stepBreaker() {
675
876
  this._consecutiveFailures++;
676
- if (misroute === null && !call.noRetry) {
677
- const status = call.status;
678
- const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : call.error !== null ? "none" : "unknown";
679
- let write = false;
680
- if (this._consecutiveFailures === 1 || this._failureKind === null) {
681
- if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
682
- this._lastFailureWarnAt = Date.now();
683
- write = true;
684
- }
685
- this._failureKind = kind;
686
- } else if (kind !== this._failureKind) {
687
- this._failureKind = kind;
688
- write = true;
689
- }
690
- if (write) this._warnOutage(kind, status, call.error);
691
- }
692
877
  if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
693
878
  this._circuitOpenUntil = Date.now() + this._circuitBreakerCooldownMs;
694
879
  warn(
695
880
  `HTTP exporter circuit breaker opened after ${this._consecutiveFailures} failures. Cooldown: ${this._circuitBreakerCooldownMs / 1e3}s`
696
881
  );
697
882
  }
698
- return "failure" /* FAILURE */;
699
883
  }
700
- /** The outage WARNING for one kind of failure. */
701
- _warnOutage(kind, status, error) {
884
+ /**
885
+ * The outage WARNING: at the first failure of an outage, and when its kind changes. At most
886
+ * one in the window for the first failure; a misroute has its own WARNING, at most one in
887
+ * the window. `retrying` is true for a retryable answer (written after
888
+ * `RETRY_WARN_AFTER_ATTEMPTS` failed attempts of one batch, or when its retries end before
889
+ * that), false for an answer that a retry cannot change (one attempt). The WARNING names the last HTTP status, or the error
890
+ * code when there was no answer. It never names the key or the response body, and names
891
+ * the API key as a cause only for a 401 or 403. An ack of 0 and 0 has its WARNING from
892
+ * `checkAck`.
893
+ */
894
+ _warnOutage(answer, retrying) {
895
+ if (answer.misroute !== null) {
896
+ if (Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
897
+ this._lastMisrouteWarnAt = Date.now();
898
+ warn(
899
+ `Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${answer.misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
900
+ );
901
+ }
902
+ return;
903
+ }
904
+ if (answer.zeroAck) return;
905
+ const status = answer.status;
906
+ const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : answer.error !== null ? "none" : "unknown";
907
+ let write = false;
908
+ if (this._failureKind === null || !retrying && this._consecutiveFailures === 1) {
909
+ if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
910
+ this._lastFailureWarnAt = Date.now();
911
+ write = true;
912
+ }
913
+ this._failureKind = kind;
914
+ } else if (kind !== this._failureKind) {
915
+ this._failureKind = kind;
916
+ write = true;
917
+ }
918
+ if (!write) return;
702
919
  const where = `Risicare: spans are NOT reaching ${this._safeEndpoint}`;
920
+ if (retrying) {
921
+ let limit = `${this._retryHorizonS} s from its first attempt (retryHorizonS)`;
922
+ if (this._maxRetries !== void 0) limit = `${this._maxRetries} attempt(s) (maxRetries) or ${limit}`;
923
+ const retryText = `The SDK sends each batch again with a growing wait, for up to ${limit}, and then drops it (reason=retry_exhausted).`;
924
+ if (kind === "http") {
925
+ warn(`${where}: it answered HTTP ${status}. ${retryText} Set debug: true to see each attempt.`);
926
+ } else {
927
+ warn(
928
+ `${where}: no answer (${answer.error}). The endpoint cannot be reached, or it did not answer in time. ${retryText} Set debug: true to see each attempt.`
929
+ );
930
+ }
931
+ return;
932
+ }
703
933
  const tail = "Tracing continues, but the SDK drops the spans that it cannot send.";
704
934
  if (kind === "auth") {
705
935
  warn(
@@ -707,15 +937,11 @@ var HttpExporter = class {
707
937
  );
708
938
  } else if (kind === "http") {
709
939
  warn(
710
- `${where}: it answered HTTP ${status}. The server did not take the batch. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
940
+ `${where}: it answered HTTP ${status}. The server did not take the batch, and a retry cannot change that answer. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
711
941
  );
712
942
  } else if (kind === "none" && this._endpointHasCredentials) {
713
943
  warn(
714
- `${where}: no answer (${error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
715
- );
716
- } else if (kind === "none") {
717
- warn(
718
- `${where}: no answer (${error}). The endpoint cannot be reached, or it did not answer in time. Check the endpoint and the network. ${tail} Set debug: true to see each attempt.`
944
+ `${where}: no answer (${answer.error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
719
945
  );
720
946
  } else {
721
947
  warn(
@@ -723,33 +949,38 @@ var HttpExporter = class {
723
949
  );
724
950
  }
725
951
  }
952
+ /** The WARNING for spans dropped as `retry_exhausted`. At most one in the window. */
953
+ _warnExhausted(nSpans, attempts, answer) {
954
+ if (Date.now() - this._lastExhaustedWarnAt <= FAILURE_WARN_WINDOW_MS) return;
955
+ this._lastExhaustedWarnAt = Date.now();
956
+ const last = answer.status !== null ? `HTTP ${answer.status}` : `no answer (${answer.error})`;
957
+ const capacity = answer.status !== null && CAPACITY_REFUSAL_STATUSES.has(answer.status) ? " (over capacity)" : "";
958
+ const limit = this._maxRetries !== void 0 && attempts >= this._maxRetries ? `the maxRetries limit of ${this._maxRetries} attempt(s)` : `the retry horizon of ${this._retryHorizonS} s (retryHorizonS)`;
959
+ warn(
960
+ `Risicare: ${nSpans} span(s) DISCARDED (reason=retry_exhausted): ${this._safeEndpoint} did not take them in ${attempts} attempt(s), and ${limit} is used up. The last answer: ${last}${capacity}. A gateway outage that is longer than the retry horizon loses spans. To wait longer, raise retryHorizonS.`
961
+ );
962
+ }
963
+ /** The WARNING for spans dropped as `too_large`. One time for each exporter. */
964
+ _warnTooLarge(nSpans) {
965
+ if (this._tooLargeWarned) return;
966
+ this._tooLargeWarned = true;
967
+ warn(
968
+ `Risicare: ${nSpans} span(s) DISCARDED (reason=too_large): ${this._safeEndpoint} answered HTTP 413 (the request is too large) for a batch, and again for a half of it after one split. The SDK does not send these spans again. Make the span attributes smaller. This WARNING is written one time; droppedSpansByReason.too_large has the count.`
969
+ );
970
+ }
726
971
  /** Record a MISROUTE for the retry loop and the breaker. Always returns FAILURE. */
727
972
  _noteMisroute(call, answer) {
728
973
  call.misroute = this._scrub(answer, 300);
729
974
  debug(`Ingest misroute: ${call.misroute}`);
730
975
  return "failure" /* FAILURE */;
731
976
  }
732
- /**
733
- * Record a CAPACITY REFUSAL for the retry loop. Returns true if it was one.
734
- *
735
- * `Retry-After` is honoured only in its delta-seconds form (RFC 9110 §10.2.3 also permits an
736
- * HTTP-date); anything else falls back to our own backoff rather than guessing at a date
737
- * parse.
738
- */
739
- _noteRefusal(call, status, retryAfterRaw) {
740
- if (!CAPACITY_REFUSAL_STATUSES.has(status)) return false;
741
- let delay = REFUSAL_NO_HINT;
742
- if (retryAfterRaw) {
743
- const secs = Number(retryAfterRaw);
744
- if (Number.isFinite(secs) && secs >= 0) {
745
- delay = Math.min(secs * 1e3, RETRY_AFTER_CEILING_MS);
746
- }
747
- }
748
- call.refusalAfterMs = delay;
749
- debug(`Ingest refused for capacity: HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
750
- return true;
977
+ /** Record the `Retry-After` wait of a retryable answer for the retry loop. */
978
+ _noteRetryAfter(call, status, retryAfterRaw) {
979
+ if (!RETRY_STATUSES.has(status)) return;
980
+ call.retryAfterS = parseRetryAfterS(retryAfterRaw);
981
+ debug(`Ingest answered HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
751
982
  }
752
- async _sendRequest(body, spansSent, call, spans) {
983
+ async _sendRequest(body, spansSent, call) {
753
984
  const cancelled = call.cancel.signal;
754
985
  const timeoutMs = this._timeoutMs;
755
986
  const url = `${this._endpoint}/v1/spans`;
@@ -817,14 +1048,15 @@ var HttpExporter = class {
817
1048
  } else if (ack.result === "success" /* SUCCESS */) {
818
1049
  this._quota.accepted(`the ingest endpoint ${this._safeEndpoint}`);
819
1050
  }
820
- if (ack.result === "success" /* SUCCESS */) this._acks.set(spans, ack.counts);
1051
+ if (ack.result === "success" /* SUCCESS */) call.ack = ack.counts;
821
1052
  if (ack.final) call.noRetry = true;
822
1053
  return ack.result;
823
- } else if (response.status === 408 || response.status === 504) {
1054
+ }
1055
+ this._noteRetryAfter(call, response.status, response.headers?.get?.("Retry-After") ?? null);
1056
+ if (response.status === 408 || response.status === 504) {
824
1057
  debug(`HTTP export timeout: ${response.status}`);
825
1058
  return "timeout" /* TIMEOUT */;
826
1059
  }
827
- this._noteRefusal(call, response.status, response.headers.get("Retry-After"));
828
1060
  debug(`HTTP export failed: ${response.status}`);
829
1061
  return "failure" /* FAILURE */;
830
1062
  } catch (e) {
@@ -838,20 +1070,9 @@ var HttpExporter = class {
838
1070
  }
839
1071
  }
840
1072
  shutdown() {
1073
+ this._noWait = false;
841
1074
  }
842
1075
  };
843
- function sleep(ms, signal) {
844
- return new Promise((resolve) => {
845
- if (signal?.aborted) return resolve();
846
- const done = () => {
847
- clearTimeout(timer);
848
- signal?.removeEventListener("abort", done);
849
- resolve();
850
- };
851
- const timer = setTimeout(done, ms);
852
- signal?.addEventListener("abort", done, { once: true });
853
- });
854
- }
855
1076
  function asCount(v) {
856
1077
  return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : void 0;
857
1078
  }