risicare 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +76 -5
  2. package/README.md +157 -39
  3. package/dist/frameworks/instructor.cjs +338 -117
  4. package/dist/frameworks/instructor.cjs.map +1 -1
  5. package/dist/frameworks/instructor.js +338 -117
  6. package/dist/frameworks/instructor.js.map +1 -1
  7. package/dist/frameworks/langchain.cjs +351 -130
  8. package/dist/frameworks/langchain.cjs.map +1 -1
  9. package/dist/frameworks/langchain.js +351 -130
  10. package/dist/frameworks/langchain.js.map +1 -1
  11. package/dist/frameworks/langgraph.cjs +351 -130
  12. package/dist/frameworks/langgraph.cjs.map +1 -1
  13. package/dist/frameworks/langgraph.js +351 -130
  14. package/dist/frameworks/langgraph.js.map +1 -1
  15. package/dist/frameworks/llamaindex.cjs +351 -130
  16. package/dist/frameworks/llamaindex.cjs.map +1 -1
  17. package/dist/frameworks/llamaindex.js +351 -130
  18. package/dist/frameworks/llamaindex.js.map +1 -1
  19. package/dist/index.cjs +808 -226
  20. package/dist/index.cjs.map +1 -1
  21. package/dist/index.d.cts +137 -21
  22. package/dist/index.d.ts +137 -21
  23. package/dist/index.js +804 -226
  24. package/dist/index.js.map +1 -1
  25. package/dist/providers/anthropic/index.cjs +351 -130
  26. package/dist/providers/anthropic/index.cjs.map +1 -1
  27. package/dist/providers/anthropic/index.js +351 -130
  28. package/dist/providers/anthropic/index.js.map +1 -1
  29. package/dist/providers/bedrock/index.cjs +346 -125
  30. package/dist/providers/bedrock/index.cjs.map +1 -1
  31. package/dist/providers/bedrock/index.js +346 -125
  32. package/dist/providers/bedrock/index.js.map +1 -1
  33. package/dist/providers/cerebras/index.cjs +351 -130
  34. package/dist/providers/cerebras/index.cjs.map +1 -1
  35. package/dist/providers/cerebras/index.js +351 -130
  36. package/dist/providers/cerebras/index.js.map +1 -1
  37. package/dist/providers/cohere/index.cjs +351 -130
  38. package/dist/providers/cohere/index.cjs.map +1 -1
  39. package/dist/providers/cohere/index.js +351 -130
  40. package/dist/providers/cohere/index.js.map +1 -1
  41. package/dist/providers/google/index.cjs +351 -130
  42. package/dist/providers/google/index.cjs.map +1 -1
  43. package/dist/providers/google/index.js +351 -130
  44. package/dist/providers/google/index.js.map +1 -1
  45. package/dist/providers/groq/index.cjs +351 -130
  46. package/dist/providers/groq/index.cjs.map +1 -1
  47. package/dist/providers/groq/index.js +351 -130
  48. package/dist/providers/groq/index.js.map +1 -1
  49. package/dist/providers/huggingface/index.cjs +351 -130
  50. package/dist/providers/huggingface/index.cjs.map +1 -1
  51. package/dist/providers/huggingface/index.js +351 -130
  52. package/dist/providers/huggingface/index.js.map +1 -1
  53. package/dist/providers/mistral/index.cjs +351 -130
  54. package/dist/providers/mistral/index.cjs.map +1 -1
  55. package/dist/providers/mistral/index.js +351 -130
  56. package/dist/providers/mistral/index.js.map +1 -1
  57. package/dist/providers/ollama/index.cjs +351 -130
  58. package/dist/providers/ollama/index.cjs.map +1 -1
  59. package/dist/providers/ollama/index.js +351 -130
  60. package/dist/providers/ollama/index.js.map +1 -1
  61. package/dist/providers/openai/index.cjs +351 -130
  62. package/dist/providers/openai/index.cjs.map +1 -1
  63. package/dist/providers/openai/index.js +351 -130
  64. package/dist/providers/openai/index.js.map +1 -1
  65. package/dist/providers/together/index.cjs +351 -130
  66. package/dist/providers/together/index.cjs.map +1 -1
  67. package/dist/providers/together/index.js +351 -130
  68. package/dist/providers/together/index.js.map +1 -1
  69. package/dist/providers/vercel-ai/index.cjs +351 -130
  70. package/dist/providers/vercel-ai/index.cjs.map +1 -1
  71. package/dist/providers/vercel-ai/index.js +351 -130
  72. package/dist/providers/vercel-ai/index.js.map +1 -1
  73. package/package.json +1 -1
@@ -1,6 +1,35 @@
1
1
  // src/client.ts
2
2
  import { createHash as createHash2 } from "crypto";
3
3
 
4
+ // src/exporters/retry.ts
5
+ var RETRY_STATUSES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
6
+ var TOO_LARGE_STATUS = 413;
7
+ var RETRY_BACKOFF_BASE_S = 1;
8
+ var RETRY_BACKOFF_CAP_S = 30;
9
+ var DEFAULT_RETRY_HORIZON_S = 300;
10
+ var DEFAULT_SHUTDOWN_TIMEOUT_S = 10;
11
+ var RETRY_WARN_AFTER_ATTEMPTS = 3;
12
+ var jitter = { fraction: () => Math.random() };
13
+ function backoffDelayS(retryNumber, fraction) {
14
+ const exponent = Math.min(Math.max(retryNumber, 0), 62);
15
+ const ceiling = Math.min(RETRY_BACKOFF_CAP_S, RETRY_BACKOFF_BASE_S * 2 ** exponent);
16
+ return Math.min(Math.max(fraction, 0), 1) * ceiling;
17
+ }
18
+ var SECONDS = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
19
+ function parseRetryAfterS(value, nowMs = Date.now()) {
20
+ if (value === null || value === void 0) return null;
21
+ const text = String(value).trim();
22
+ if (text === "") return null;
23
+ if (SECONDS.test(text)) {
24
+ const seconds = Number(text);
25
+ return Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
26
+ }
27
+ if (!/[A-Za-z]/.test(text)) return null;
28
+ const when = Date.parse(text);
29
+ if (!Number.isFinite(when)) return null;
30
+ return Math.max(0, (when - nowMs) / 1e3);
31
+ }
32
+
4
33
  // src/globals.ts
5
34
  import { AsyncLocalStorage } from "async_hooks";
6
35
  var G = globalThis;
@@ -170,10 +199,24 @@ var VALUE_RULE_PREFIXES = new Set(
170
199
  );
171
200
 
172
201
  // src/exporters/batch.ts
202
+ var SHUTDOWN_BUDGET_MS = DEFAULT_SHUTDOWN_TIMEOUT_S * 1e3;
173
203
  var MAX_TIMEOUT_MS = 2 ** 31 - 1;
174
204
  var MAX_BATCH_BYTES = 8 * 1024 * 1024;
175
205
  var utf8ByteLength = typeof Buffer !== "undefined" && typeof Buffer.byteLength === "function" ? (s) => Buffer.byteLength(s, "utf8") : (s) => s.length * 3;
176
206
 
207
+ // src/context/storage.ts
208
+ function storage() {
209
+ return getContextStorage();
210
+ }
211
+ function getContext() {
212
+ return storage().getStore() ?? {};
213
+ }
214
+ function runWithContext(overrides, fn) {
215
+ const parent = getContext();
216
+ const merged = { ...parent, ...overrides };
217
+ return storage().run(merged, fn);
218
+ }
219
+
177
220
  // src/ids.ts
178
221
  import { randomBytes } from "crypto";
179
222
 
@@ -478,19 +521,6 @@ function classifyError(error) {
478
521
  return CANONICAL_ERROR_CODES.has(GENERIC_FALLBACK) ? GENERIC_FALLBACK : "";
479
522
  }
480
523
 
481
- // src/context/storage.ts
482
- function storage() {
483
- return getContextStorage();
484
- }
485
- function getContext() {
486
- return storage().getStore() ?? {};
487
- }
488
- function runWithContext(overrides, fn) {
489
- const parent = getContext();
490
- const merged = { ...parent, ...overrides };
491
- return storage().run(merged, fn);
492
- }
493
-
494
524
  // src/tracer.ts
495
525
  function attachErrorCode(span, error) {
496
526
  try {
@@ -505,7 +535,7 @@ function attachErrorCode(span, error) {
505
535
  }
506
536
 
507
537
  // src/version.ts
508
- var SDK_VERSION = "0.10.0";
538
+ var SDK_VERSION = "0.11.0";
509
539
 
510
540
  // src/exporters/quota.ts
511
541
  var QUOTA_REASON = "monthly_quota_exceeded";
@@ -613,12 +643,18 @@ var QuotaPause = class {
613
643
 
614
644
  // src/exporters/http.ts
615
645
  var CAPACITY_REFUSAL_STATUSES = /* @__PURE__ */ new Set([429, 503]);
616
- var RETRY_AFTER_CEILING_MS = 1e4;
617
- var REFUSAL_BACKOFF_BASE_MS = 1e3;
618
- var REFUSAL_NO_HINT = -1;
619
646
  var MISROUTE_WARN_WINDOW_MS = 3e5;
620
647
  var AUTH_REFUSAL_STATUSES = /* @__PURE__ */ new Set([401, 403]);
621
648
  var FAILURE_WARN_WINDOW_MS = 3e5;
649
+ var MAX_TIMER_MS = 2 ** 31 - 1;
650
+ function validHorizonS(value) {
651
+ if (value === void 0) return DEFAULT_RETRY_HORIZON_S;
652
+ if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
653
+ warn(
654
+ `Risicare: retryHorizonS=${String(value)} is not a finite number of 0 or more; using the default, ${DEFAULT_RETRY_HORIZON_S} s.`
655
+ );
656
+ return DEFAULT_RETRY_HORIZON_S;
657
+ }
622
658
  var HttpExporter = class {
623
659
  name = "http";
624
660
  _endpoint;
@@ -628,20 +664,28 @@ var HttpExporter = class {
628
664
  _timeoutMs;
629
665
  _maxRetries;
630
666
  _compress;
667
+ _retryHorizonS;
668
+ /** Total HTTP requests that this exporter sent (each retry and each half of a split counts). */
669
+ attempts = 0;
631
670
  // Circuit breaker (CON-201 / B-16). Threshold and cooldown sized for
632
- // a "30-second prod restart" scenario: with default 1s batch flush a
633
- // 5-failure threshold opens the circuit ~5s into an outage and the 60s
634
- // cooldown gives the gateway time to come back before the half-open
635
- // probe fires. Lower numbers (was 3@30s) tripped on transient blips.
671
+ // a "30-second prod restart" scenario. Since W2-1c only answers that a retry
672
+ // cannot change move it (see the block above).
636
673
  _consecutiveFailures = 0;
637
674
  _lastMisrouteWarnAt = -Infinity;
638
675
  _lastFailureWarnAt = -Infinity;
676
+ _lastExhaustedWarnAt = -Infinity;
677
+ /** "log once" (W2-1c): the WARNING for a `too_large` drop is written one time. */
678
+ _tooLargeWarned = false;
639
679
  /**
640
680
  * The kind of failure that the last outage WARNING named ('auth', 'http', 'none',
641
681
  * 'unknown'). `null` after a success. A new kind in the same outage writes one more
642
682
  * WARNING, also inside the window.
643
683
  */
644
684
  _failureKind = null;
685
+ /** `endRetries()` was called, and `shutdown()` has not run since. */
686
+ _noWait = false;
687
+ /** The wake-ups of the backoff waits in progress. `endRetries()` calls each one. */
688
+ _waits = /* @__PURE__ */ new Set();
645
689
  /** The endpoint as a log line may show it: no userinfo, no query (see `safeUrl`). */
646
690
  _safeEndpoint;
647
691
  /** The endpoint holds userinfo. `fetch` refuses such a URL. */
@@ -652,6 +696,8 @@ var HttpExporter = class {
652
696
  _acks = /* @__PURE__ */ new WeakMap();
653
697
  /** The requests that each finished call sent, by its batch. Read once by `takeAttempts(spans)`. */
654
698
  _requests = /* @__PURE__ */ new WeakMap();
699
+ /** The drop reason of each failed call, by its batch. Read once by `takeDropReason(spans)`. */
700
+ _dropReasons = /* @__PURE__ */ new WeakMap();
655
701
  /** The batch processor dropped span(s) from a full queue in the quota pause (S-1). */
656
702
  noteQuotaDrop(count = 1) {
657
703
  this._quota.addDrops(count);
@@ -676,8 +722,16 @@ var HttpExporter = class {
676
722
  this._projectId = options.projectId;
677
723
  this._environment = options.environment;
678
724
  this._timeoutMs = options.timeoutMs ?? 5e3;
679
- this._maxRetries = options.maxRetries ?? 3;
725
+ let maxRetries = options.maxRetries;
726
+ if (maxRetries !== void 0 && !(Number.isInteger(maxRetries) && maxRetries >= 1)) {
727
+ warn(
728
+ `Risicare: maxRetries=${String(maxRetries)} is not a whole number of 1 or more; the exporter uses no count limit (the retry horizon is the limit).`
729
+ );
730
+ maxRetries = void 0;
731
+ }
732
+ this._maxRetries = maxRetries;
680
733
  this._compress = options.compress ?? false;
734
+ this._retryHorizonS = validHorizonS(options.retryHorizonS);
681
735
  this._safeEndpoint = safeUrl(this._endpoint);
682
736
  this._endpointHasCredentials = hasCredentials(this._endpoint);
683
737
  }
@@ -699,16 +753,18 @@ var HttpExporter = class {
699
753
  [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
700
754
  return `HttpExporter { endpoint: '${this._safeEndpoint}' }`;
701
755
  }
702
- /** The most requests that one `export()` call sends (`maxRetries`). */
756
+ /**
757
+ * The most attempts for one request body (`maxRetries`). `undefined` when no count is set:
758
+ * the retry horizon is the limit.
759
+ */
703
760
  get attemptsPerExport() {
704
761
  return this._maxRetries;
705
762
  }
706
763
  /**
707
764
  * Returns the number of HTTP requests that the `export()` call for `spans` sent, and clears
708
- * it. This is the number that was sent, not the most that the call may send
709
- * (`attemptsPerExport`): a 401 or 403 stops the loop after one request, and an open circuit
710
- * breaker sends none. Returns `null` when no call is on record for `spans`. The answer is
711
- * kept per batch, as `takeAck` keeps the ack.
765
+ * it. Each retry and each half of a 413 split counts one; a call that the open circuit
766
+ * breaker or the quota pause stops sends none. Returns `null` when no call is on record for
767
+ * `spans`. The answer is kept per batch, as `takeAck` keeps the ack.
712
768
  */
713
769
  takeAttempts(spans) {
714
770
  const n = this._requests.get(spans) ?? null;
@@ -725,6 +781,12 @@ var HttpExporter = class {
725
781
  this._acks.delete(spans);
726
782
  return ack;
727
783
  }
784
+ /** Returns why the `export()` call for `spans` failed (W2-1c), and clears it. See `SpanExporter`. */
785
+ takeDropReason(spans) {
786
+ const reason = this._dropReasons.get(spans) ?? null;
787
+ this._dropReasons.delete(spans);
788
+ return reason;
789
+ }
728
790
  /** Returns the spans of `spans` that could not be serialized and were not sent, and clears them. */
729
791
  takeUnserializable(spans) {
730
792
  const bad = this._unserializable.get(spans) ?? [];
@@ -733,7 +795,7 @@ var HttpExporter = class {
733
795
  }
734
796
  /**
735
797
  * Cancel the `export()` call for `spans` (all calls when `spans` is not given): its
736
- * request, its retries and its backoff sleep. The batch processor calls it when
798
+ * request, its retries and its backoff wait. The batch processor calls it when
737
799
  * `shutdown()` abandons the batch, so that the call does not keep the process alive. The
738
800
  * cancelled call returns FAILURE, changes no counter (the circuit breaker included) and
739
801
  * logs no WARNING.
@@ -743,24 +805,39 @@ var HttpExporter = class {
743
805
  else for (const call of this._calls.values()) call.cancel.abort();
744
806
  }
745
807
  /**
746
- * Send one batch. The limits are the request timeout and `maxRetries`.
808
+ * End the retry waits of this exporter until its next `shutdown()`. The batch processor
809
+ * calls it when `shutdown()` starts. A call that waits for a retry makes one more attempt
810
+ * at once. An attempt that fails after this is not retried: the call fails with the drop
811
+ * reason `shutdown_residue`. So a drain against a dead endpoint costs one attempt for each
812
+ * batch, not the shutdown budget.
813
+ */
814
+ endRetries() {
815
+ this._noWait = true;
816
+ for (const wake of [...this._waits]) wake();
817
+ }
818
+ /**
819
+ * Send one batch, with the W2-1c retry policy (see the block above the class).
820
+ *
821
+ * Resolves SUCCESS when the backend accounted for the batch: `takeAck` has the counts (and
822
+ * the spans that a 413 split dropped, in `dropped`). Resolves FAILURE when no span was
823
+ * accounted for: `takeDropReason` then says why, and the batch processor drops the batch
824
+ * with that reason.
747
825
  *
748
826
  * ⚠ NO CALLER BUDGET. `flush(timeoutMs)` only stops WAITING at its deadline; it never
749
- * cuts a request that is already sent. A short flush budget that cut the request made a
750
- * healthy, slow server look like a failure: the span used a retry, the breaker moved, and
751
- * the server stored the same spans again. Only `abort()` stops the call, and only
827
+ * cuts a request that is already sent. Only `abort()` stops the call, and only
752
828
  * `shutdown()` calls it.
753
829
  */
754
830
  async export(spans) {
755
831
  if (spans.length === 0) return "success" /* SUCCESS */;
756
832
  const call = {
757
833
  cancel: new AbortController(),
758
- refusalAfterMs: null,
834
+ retryAfterS: null,
759
835
  misroute: null,
760
836
  noRetry: false,
761
837
  status: null,
762
838
  error: null,
763
- attempts: 0
839
+ attempts: 0,
840
+ ack: null
764
841
  };
765
842
  this._calls.set(spans, call);
766
843
  try {
@@ -797,83 +874,236 @@ var HttpExporter = class {
797
874
  );
798
875
  }
799
876
  if (parts.length === 0) return "success" /* SUCCESS */;
800
- const payload = `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
801
- const now = Date.now();
802
- let isHalfOpen = false;
803
- if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
804
- if (now < this._circuitOpenUntil) {
805
- return "failure" /* FAILURE */;
877
+ if (this._consecutiveFailures >= this._circuitBreakerThreshold && Date.now() < this._circuitOpenUntil) {
878
+ this._dropReasons.set(spans, "transport_refused");
879
+ return "failure" /* FAILURE */;
880
+ }
881
+ const out = { delivered: [], dropped: {}, failed: [], cancelled: false };
882
+ const start = performance.now();
883
+ await this._deliver(parts, start, call, out, true);
884
+ if (cancelled.aborted || out.cancelled) {
885
+ this._acks.delete(spans);
886
+ return "failure" /* FAILURE */;
887
+ }
888
+ return this._finish(spans, out, call.attempts);
889
+ }
890
+ /** The request body for `parts` (the serialized spans of one part of a batch). */
891
+ _payload(parts) {
892
+ return `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
893
+ }
894
+ /**
895
+ * Send one part of a batch with the retry policy, and record its outcome in `out`.
896
+ * `maySplit` is true only for the whole batch: a 413 splits it in two one time. A half
897
+ * that gets 413 again is dropped as `too_large`.
898
+ */
899
+ async _deliver(parts, start, call, out, maySplit) {
900
+ const n = parts.length;
901
+ if (!maySplit && this._quota.paused()) {
902
+ out.delivered.push({ n, ack: { accepted: 0, rejected: 0, quotaDropped: n } });
903
+ return;
904
+ }
905
+ const result = await this._sendWithRetries(this._payload(parts), n, start, call);
906
+ if (result.kind === "delivered") {
907
+ out.delivered.push({ n, ack: result.ack });
908
+ } else if (result.kind === "too_large") {
909
+ if (maySplit && n > 1) {
910
+ const mid = Math.ceil(n / 2);
911
+ debug(`Ingest answered HTTP 413 for ${n} span(s): sending two halves of ${mid} and ${n - mid}`);
912
+ await this._deliver(parts.slice(0, mid), start, call, out, false);
913
+ if (!out.cancelled) await this._deliver(parts.slice(mid), start, call, out, false);
914
+ } else {
915
+ out.dropped.too_large = (out.dropped.too_large ?? 0) + n;
806
916
  }
807
- isHalfOpen = true;
917
+ } else if (result.kind === "cancelled") {
918
+ out.cancelled = true;
919
+ } else {
920
+ out.failed.push({ n, reason: result.kind, answer: result.answer });
808
921
  }
809
- const maxAttempts = isHalfOpen ? 1 : this._maxRetries;
810
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
811
- call.refusalAfterMs = null;
922
+ }
923
+ /** Send one request body until it is delivered or the policy stops. */
924
+ async _sendWithRetries(body, spansSent, start, call) {
925
+ const cancelled = call.cancel.signal;
926
+ let retryNumber = 0;
927
+ let attemptsHere = 0;
928
+ for (; ; ) {
929
+ if (cancelled.aborted) return { kind: "cancelled" };
930
+ call.retryAfterS = null;
812
931
  call.misroute = null;
813
932
  call.noRetry = false;
814
933
  call.status = null;
815
934
  call.error = null;
935
+ call.ack = null;
816
936
  call.attempts++;
817
- const result = await this._sendRequest(payload, parts.length, call, spans);
818
- if (cancelled.aborted) {
819
- this._acks.delete(spans);
820
- return "failure" /* FAILURE */;
821
- }
822
- if (result === "success" /* SUCCESS */) {
823
- this._consecutiveFailures = 0;
824
- this._failureKind = null;
825
- return result;
937
+ attemptsHere++;
938
+ this.attempts++;
939
+ const result = await this._sendRequest(body, spansSent, call);
940
+ if (cancelled.aborted) return { kind: "cancelled" };
941
+ if (result === "success" /* SUCCESS */) return { kind: "delivered", ack: call.ack };
942
+ if (call.status === TOO_LARGE_STATUS) return { kind: "too_large" };
943
+ const answer = {
944
+ status: call.status,
945
+ error: call.error,
946
+ misroute: call.misroute,
947
+ zeroAck: call.noRetry,
948
+ sent: spansSent
949
+ };
950
+ if (!this._retryable(call)) return { kind: "transport_refused", answer };
951
+ if (attemptsHere === RETRY_WARN_AFTER_ATTEMPTS) this._warnOutage(answer, true);
952
+ if (this._noWait) return { kind: "shutdown_residue", answer };
953
+ if (this._maxRetries !== void 0 && attemptsHere >= this._maxRetries) {
954
+ return { kind: "retry_exhausted", answer };
826
955
  }
827
- if (call.misroute !== null || call.noRetry) break;
828
- if (call.status !== null && AUTH_REFUSAL_STATUSES.has(call.status)) break;
829
- if (attempt < maxAttempts - 1) {
830
- const refusal = call.refusalAfterMs;
831
- const delay = refusal === null ? 100 * Math.pow(2, attempt) : refusal === REFUSAL_NO_HINT ? REFUSAL_BACKOFF_BASE_MS * Math.pow(2, attempt) : refusal;
832
- await sleep(delay, cancelled);
833
- if (cancelled.aborted) return "failure" /* FAILURE */;
956
+ const remainingS = this._retryHorizonS - (performance.now() - start) / 1e3;
957
+ if (remainingS <= 0) return { kind: "retry_exhausted", answer };
958
+ const hint = call.retryAfterS;
959
+ const waitS = hint !== null && hint > 0 ? hint : backoffDelayS(retryNumber, jitter.fraction());
960
+ const delayS = Math.min(waitS, remainingS);
961
+ retryNumber++;
962
+ debug(
963
+ `Ingest attempt ${attemptsHere} failed (${answer.status ?? answer.error ?? "unknown"}); next attempt in ${delayS.toFixed(2)} s (${Math.round(remainingS)} s of the horizon left)`
964
+ );
965
+ await this._retryWait(delayS * 1e3, cancelled);
966
+ }
967
+ }
968
+ /** True when the answer of the last attempt of `call` can change on a retry. */
969
+ _retryable(call) {
970
+ if (call.misroute !== null || call.noRetry) return false;
971
+ if (call.status !== null) return RETRY_STATUSES.has(call.status);
972
+ if (this._endpointHasCredentials) return false;
973
+ return call.error !== null;
974
+ }
975
+ /**
976
+ * Wait `ms` before the next attempt. `abort()` and `endRetries()` end the wait at once. The
977
+ * timer does not keep the process alive: when nothing else runs, `beforeExit` starts the
978
+ * SDK's `shutdown()`, which ends the wait.
979
+ */
980
+ _retryWait(ms, signal) {
981
+ return new Promise((resolve) => {
982
+ if (signal.aborted || this._noWait || ms <= 0) return resolve();
983
+ const done = () => {
984
+ clearTimeout(timer);
985
+ signal.removeEventListener("abort", done);
986
+ this._waits.delete(done);
987
+ resolve();
988
+ };
989
+ const timer = setTimeout(done, Math.min(ms, MAX_TIMER_MS));
990
+ timer.unref?.();
991
+ signal.addEventListener("abort", done, { once: true });
992
+ this._waits.add(done);
993
+ });
994
+ }
995
+ /** Turn the outcome of the parts of one batch into the result of `export()`. */
996
+ _finish(spans, out, attempts) {
997
+ const reasons = new Set(out.failed.map((f) => f.reason));
998
+ if (out.delivered.length === 0 && Object.keys(out.dropped).length === 0 && reasons.size === 1) {
999
+ const reason = out.failed[0].reason;
1000
+ const nFailed = out.failed.reduce((sum, f) => sum + f.n, 0);
1001
+ this._dropReasons.set(spans, reason);
1002
+ return this._fail(reason, out.failed[out.failed.length - 1].answer, nFailed, attempts);
1003
+ }
1004
+ let ack;
1005
+ if (out.delivered.length === 1 && Object.keys(out.dropped).length === 0 && out.failed.length === 0) {
1006
+ ack = out.delivered[0].ack;
1007
+ } else {
1008
+ let accepted = 0;
1009
+ let rejected = 0;
1010
+ let quotaDropped = 0;
1011
+ for (const part of out.delivered) {
1012
+ accepted += part.ack !== null ? part.ack.accepted : part.n;
1013
+ rejected += part.ack !== null ? part.ack.rejected : 0;
1014
+ quotaDropped += part.ack?.quotaDropped ?? 0;
834
1015
  }
1016
+ const dropped = { ...out.dropped };
1017
+ for (const f of out.failed) dropped[f.reason] = (dropped[f.reason] ?? 0) + f.n;
1018
+ ack = { accepted, rejected };
1019
+ if (quotaDropped > 0) ack.quotaDropped = quotaDropped;
1020
+ if (Object.keys(dropped).length > 0) ack.dropped = dropped;
835
1021
  }
836
- if (call.refusalAfterMs !== null) {
837
- warn(
838
- `Risicare: the ingest endpoint refused this batch for capacity (HTTP 429/503) after ${call.attempts} attempt(s). This is a healthy server shedding load, not an outage. This export of the batch failed; the circuit breaker does not count it. Reduce batch size or export rate if it persists.`
839
- );
1022
+ if (ack !== null) this._acks.set(spans, ack);
1023
+ this._consecutiveFailures = 0;
1024
+ this._failureKind = null;
1025
+ const tooLarge = out.dropped.too_large ?? 0;
1026
+ if (tooLarge > 0) this._warnTooLarge(tooLarge);
1027
+ for (const f of out.failed) {
1028
+ if (f.reason === "retry_exhausted") this._warnExhausted(f.n, attempts, f.answer);
1029
+ }
1030
+ return "success" /* SUCCESS */;
1031
+ }
1032
+ /** End a call in which no span was accounted for. Always returns FAILURE. */
1033
+ _fail(reason, answer, nSpans, attempts) {
1034
+ if (reason === "shutdown_residue") {
1035
+ debug(`Export ended by shutdown after ${attempts} attempt(s)`);
840
1036
  return "failure" /* FAILURE */;
841
1037
  }
842
- const misroute = call.misroute;
843
- if (misroute !== null && Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
844
- this._lastMisrouteWarnAt = Date.now();
845
- warn(
846
- `Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
847
- );
1038
+ if (reason === "retry_exhausted") {
1039
+ if (answer.status === null || !CAPACITY_REFUSAL_STATUSES.has(answer.status)) this._stepBreaker();
1040
+ this._warnOutage(answer, true);
1041
+ this._warnExhausted(nSpans, attempts, answer);
1042
+ return "failure" /* FAILURE */;
848
1043
  }
1044
+ this._stepBreaker();
1045
+ this._warnOutage(answer, false);
1046
+ return "failure" /* FAILURE */;
1047
+ }
1048
+ /** Count one failed call toward the circuit breaker. */
1049
+ _stepBreaker() {
849
1050
  this._consecutiveFailures++;
850
- if (misroute === null && !call.noRetry) {
851
- const status = call.status;
852
- const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : call.error !== null ? "none" : "unknown";
853
- let write = false;
854
- if (this._consecutiveFailures === 1 || this._failureKind === null) {
855
- if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
856
- this._lastFailureWarnAt = Date.now();
857
- write = true;
858
- }
859
- this._failureKind = kind;
860
- } else if (kind !== this._failureKind) {
861
- this._failureKind = kind;
862
- write = true;
863
- }
864
- if (write) this._warnOutage(kind, status, call.error);
865
- }
866
1051
  if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
867
1052
  this._circuitOpenUntil = Date.now() + this._circuitBreakerCooldownMs;
868
1053
  warn(
869
1054
  `HTTP exporter circuit breaker opened after ${this._consecutiveFailures} failures. Cooldown: ${this._circuitBreakerCooldownMs / 1e3}s`
870
1055
  );
871
1056
  }
872
- return "failure" /* FAILURE */;
873
1057
  }
874
- /** The outage WARNING for one kind of failure. */
875
- _warnOutage(kind, status, error) {
1058
+ /**
1059
+ * The outage WARNING: at the first failure of an outage, and when its kind changes. At most
1060
+ * one in the window for the first failure; a misroute has its own WARNING, at most one in
1061
+ * the window. `retrying` is true for a retryable answer (written after
1062
+ * `RETRY_WARN_AFTER_ATTEMPTS` failed attempts of one batch, or when its retries end before
1063
+ * that), false for an answer that a retry cannot change (one attempt). The WARNING names the last HTTP status, or the error
1064
+ * code when there was no answer. It never names the key or the response body, and names
1065
+ * the API key as a cause only for a 401 or 403. An ack of 0 and 0 has its WARNING from
1066
+ * `checkAck`.
1067
+ */
1068
+ _warnOutage(answer, retrying) {
1069
+ if (answer.misroute !== null) {
1070
+ if (Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
1071
+ this._lastMisrouteWarnAt = Date.now();
1072
+ warn(
1073
+ `Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${answer.misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
1074
+ );
1075
+ }
1076
+ return;
1077
+ }
1078
+ if (answer.zeroAck) return;
1079
+ const status = answer.status;
1080
+ const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : answer.error !== null ? "none" : "unknown";
1081
+ let write = false;
1082
+ if (this._failureKind === null || !retrying && this._consecutiveFailures === 1) {
1083
+ if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
1084
+ this._lastFailureWarnAt = Date.now();
1085
+ write = true;
1086
+ }
1087
+ this._failureKind = kind;
1088
+ } else if (kind !== this._failureKind) {
1089
+ this._failureKind = kind;
1090
+ write = true;
1091
+ }
1092
+ if (!write) return;
876
1093
  const where = `Risicare: spans are NOT reaching ${this._safeEndpoint}`;
1094
+ if (retrying) {
1095
+ let limit = `${this._retryHorizonS} s from its first attempt (retryHorizonS)`;
1096
+ if (this._maxRetries !== void 0) limit = `${this._maxRetries} attempt(s) (maxRetries) or ${limit}`;
1097
+ const retryText = `The SDK sends each batch again with a growing wait, for up to ${limit}, and then drops it (reason=retry_exhausted).`;
1098
+ if (kind === "http") {
1099
+ warn(`${where}: it answered HTTP ${status}. ${retryText} Set debug: true to see each attempt.`);
1100
+ } else {
1101
+ warn(
1102
+ `${where}: no answer (${answer.error}). The endpoint cannot be reached, or it did not answer in time. ${retryText} Set debug: true to see each attempt.`
1103
+ );
1104
+ }
1105
+ return;
1106
+ }
877
1107
  const tail = "Tracing continues, but the SDK drops the spans that it cannot send.";
878
1108
  if (kind === "auth") {
879
1109
  warn(
@@ -881,15 +1111,11 @@ var HttpExporter = class {
881
1111
  );
882
1112
  } else if (kind === "http") {
883
1113
  warn(
884
- `${where}: it answered HTTP ${status}. The server did not take the batch. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
1114
+ `${where}: it answered HTTP ${status}. The server did not take the batch, and a retry cannot change that answer. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
885
1115
  );
886
1116
  } else if (kind === "none" && this._endpointHasCredentials) {
887
1117
  warn(
888
- `${where}: no answer (${error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
889
- );
890
- } else if (kind === "none") {
891
- warn(
892
- `${where}: no answer (${error}). The endpoint cannot be reached, or it did not answer in time. Check the endpoint and the network. ${tail} Set debug: true to see each attempt.`
1118
+ `${where}: no answer (${answer.error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
893
1119
  );
894
1120
  } else {
895
1121
  warn(
@@ -897,33 +1123,38 @@ var HttpExporter = class {
897
1123
  );
898
1124
  }
899
1125
  }
1126
+ /** The WARNING for spans dropped as `retry_exhausted`. At most one in the window. */
1127
+ _warnExhausted(nSpans, attempts, answer) {
1128
+ if (Date.now() - this._lastExhaustedWarnAt <= FAILURE_WARN_WINDOW_MS) return;
1129
+ this._lastExhaustedWarnAt = Date.now();
1130
+ const last = answer.status !== null ? `HTTP ${answer.status}` : `no answer (${answer.error})`;
1131
+ const capacity = answer.status !== null && CAPACITY_REFUSAL_STATUSES.has(answer.status) ? " (over capacity)" : "";
1132
+ const limit = this._maxRetries !== void 0 && attempts >= this._maxRetries ? `the maxRetries limit of ${this._maxRetries} attempt(s)` : `the retry horizon of ${this._retryHorizonS} s (retryHorizonS)`;
1133
+ warn(
1134
+ `Risicare: ${nSpans} span(s) DISCARDED (reason=retry_exhausted): ${this._safeEndpoint} did not take them in ${attempts} attempt(s), and ${limit} is used up. The last answer: ${last}${capacity}. A gateway outage that is longer than the retry horizon loses spans. To wait longer, raise retryHorizonS.`
1135
+ );
1136
+ }
1137
+ /** The WARNING for spans dropped as `too_large`. One time for each exporter. */
1138
+ _warnTooLarge(nSpans) {
1139
+ if (this._tooLargeWarned) return;
1140
+ this._tooLargeWarned = true;
1141
+ warn(
1142
+ `Risicare: ${nSpans} span(s) DISCARDED (reason=too_large): ${this._safeEndpoint} answered HTTP 413 (the request is too large) for a batch, and again for a half of it after one split. The SDK does not send these spans again. Make the span attributes smaller. This WARNING is written one time; droppedSpansByReason.too_large has the count.`
1143
+ );
1144
+ }
900
1145
  /** Record a MISROUTE for the retry loop and the breaker. Always returns FAILURE. */
901
1146
  _noteMisroute(call, answer) {
902
1147
  call.misroute = this._scrub(answer, 300);
903
1148
  debug(`Ingest misroute: ${call.misroute}`);
904
1149
  return "failure" /* FAILURE */;
905
1150
  }
906
- /**
907
- * Record a CAPACITY REFUSAL for the retry loop. Returns true if it was one.
908
- *
909
- * `Retry-After` is honoured only in its delta-seconds form (RFC 9110 §10.2.3 also permits an
910
- * HTTP-date); anything else falls back to our own backoff rather than guessing at a date
911
- * parse.
912
- */
913
- _noteRefusal(call, status, retryAfterRaw) {
914
- if (!CAPACITY_REFUSAL_STATUSES.has(status)) return false;
915
- let delay = REFUSAL_NO_HINT;
916
- if (retryAfterRaw) {
917
- const secs = Number(retryAfterRaw);
918
- if (Number.isFinite(secs) && secs >= 0) {
919
- delay = Math.min(secs * 1e3, RETRY_AFTER_CEILING_MS);
920
- }
921
- }
922
- call.refusalAfterMs = delay;
923
- debug(`Ingest refused for capacity: HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
924
- return true;
1151
+ /** Record the `Retry-After` wait of a retryable answer for the retry loop. */
1152
+ _noteRetryAfter(call, status, retryAfterRaw) {
1153
+ if (!RETRY_STATUSES.has(status)) return;
1154
+ call.retryAfterS = parseRetryAfterS(retryAfterRaw);
1155
+ debug(`Ingest answered HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
925
1156
  }
926
- async _sendRequest(body, spansSent, call, spans) {
1157
+ async _sendRequest(body, spansSent, call) {
927
1158
  const cancelled = call.cancel.signal;
928
1159
  const timeoutMs = this._timeoutMs;
929
1160
  const url = `${this._endpoint}/v1/spans`;
@@ -991,14 +1222,15 @@ var HttpExporter = class {
991
1222
  } else if (ack.result === "success" /* SUCCESS */) {
992
1223
  this._quota.accepted(`the ingest endpoint ${this._safeEndpoint}`);
993
1224
  }
994
- if (ack.result === "success" /* SUCCESS */) this._acks.set(spans, ack.counts);
1225
+ if (ack.result === "success" /* SUCCESS */) call.ack = ack.counts;
995
1226
  if (ack.final) call.noRetry = true;
996
1227
  return ack.result;
997
- } else if (response.status === 408 || response.status === 504) {
1228
+ }
1229
+ this._noteRetryAfter(call, response.status, response.headers?.get?.("Retry-After") ?? null);
1230
+ if (response.status === 408 || response.status === 504) {
998
1231
  debug(`HTTP export timeout: ${response.status}`);
999
1232
  return "timeout" /* TIMEOUT */;
1000
1233
  }
1001
- this._noteRefusal(call, response.status, response.headers.get("Retry-After"));
1002
1234
  debug(`HTTP export failed: ${response.status}`);
1003
1235
  return "failure" /* FAILURE */;
1004
1236
  } catch (e) {
@@ -1012,20 +1244,9 @@ var HttpExporter = class {
1012
1244
  }
1013
1245
  }
1014
1246
  shutdown() {
1247
+ this._noWait = false;
1015
1248
  }
1016
1249
  };
1017
- function sleep(ms, signal) {
1018
- return new Promise((resolve) => {
1019
- if (signal?.aborted) return resolve();
1020
- const done = () => {
1021
- clearTimeout(timer);
1022
- signal?.removeEventListener("abort", done);
1023
- resolve();
1024
- };
1025
- const timer = setTimeout(done, ms);
1026
- signal?.addEventListener("abort", done, { once: true });
1027
- });
1028
- }
1029
1250
  function asCount(v) {
1030
1251
  return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : void 0;
1031
1252
  }