risicare 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -5
- package/README.md +157 -39
- package/dist/frameworks/instructor.cjs +338 -117
- package/dist/frameworks/instructor.cjs.map +1 -1
- package/dist/frameworks/instructor.js +338 -117
- package/dist/frameworks/instructor.js.map +1 -1
- package/dist/frameworks/langchain.cjs +351 -130
- package/dist/frameworks/langchain.cjs.map +1 -1
- package/dist/frameworks/langchain.js +351 -130
- package/dist/frameworks/langchain.js.map +1 -1
- package/dist/frameworks/langgraph.cjs +351 -130
- package/dist/frameworks/langgraph.cjs.map +1 -1
- package/dist/frameworks/langgraph.js +351 -130
- package/dist/frameworks/langgraph.js.map +1 -1
- package/dist/frameworks/llamaindex.cjs +351 -130
- package/dist/frameworks/llamaindex.cjs.map +1 -1
- package/dist/frameworks/llamaindex.js +351 -130
- package/dist/frameworks/llamaindex.js.map +1 -1
- package/dist/index.cjs +808 -226
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +137 -21
- package/dist/index.d.ts +137 -21
- package/dist/index.js +804 -226
- package/dist/index.js.map +1 -1
- package/dist/providers/anthropic/index.cjs +351 -130
- package/dist/providers/anthropic/index.cjs.map +1 -1
- package/dist/providers/anthropic/index.js +351 -130
- package/dist/providers/anthropic/index.js.map +1 -1
- package/dist/providers/bedrock/index.cjs +346 -125
- package/dist/providers/bedrock/index.cjs.map +1 -1
- package/dist/providers/bedrock/index.js +346 -125
- package/dist/providers/bedrock/index.js.map +1 -1
- package/dist/providers/cerebras/index.cjs +351 -130
- package/dist/providers/cerebras/index.cjs.map +1 -1
- package/dist/providers/cerebras/index.js +351 -130
- package/dist/providers/cerebras/index.js.map +1 -1
- package/dist/providers/cohere/index.cjs +351 -130
- package/dist/providers/cohere/index.cjs.map +1 -1
- package/dist/providers/cohere/index.js +351 -130
- package/dist/providers/cohere/index.js.map +1 -1
- package/dist/providers/google/index.cjs +351 -130
- package/dist/providers/google/index.cjs.map +1 -1
- package/dist/providers/google/index.js +351 -130
- package/dist/providers/google/index.js.map +1 -1
- package/dist/providers/groq/index.cjs +351 -130
- package/dist/providers/groq/index.cjs.map +1 -1
- package/dist/providers/groq/index.js +351 -130
- package/dist/providers/groq/index.js.map +1 -1
- package/dist/providers/huggingface/index.cjs +351 -130
- package/dist/providers/huggingface/index.cjs.map +1 -1
- package/dist/providers/huggingface/index.js +351 -130
- package/dist/providers/huggingface/index.js.map +1 -1
- package/dist/providers/mistral/index.cjs +351 -130
- package/dist/providers/mistral/index.cjs.map +1 -1
- package/dist/providers/mistral/index.js +351 -130
- package/dist/providers/mistral/index.js.map +1 -1
- package/dist/providers/ollama/index.cjs +351 -130
- package/dist/providers/ollama/index.cjs.map +1 -1
- package/dist/providers/ollama/index.js +351 -130
- package/dist/providers/ollama/index.js.map +1 -1
- package/dist/providers/openai/index.cjs +351 -130
- package/dist/providers/openai/index.cjs.map +1 -1
- package/dist/providers/openai/index.js +351 -130
- package/dist/providers/openai/index.js.map +1 -1
- package/dist/providers/together/index.cjs +351 -130
- package/dist/providers/together/index.cjs.map +1 -1
- package/dist/providers/together/index.js +351 -130
- package/dist/providers/together/index.js.map +1 -1
- package/dist/providers/vercel-ai/index.cjs +351 -130
- package/dist/providers/vercel-ai/index.cjs.map +1 -1
- package/dist/providers/vercel-ai/index.js +351 -130
- package/dist/providers/vercel-ai/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -37,6 +37,35 @@ module.exports = __toCommonJS(mistral_exports);
|
|
|
37
37
|
// src/client.ts
|
|
38
38
|
var import_node_crypto3 = require("crypto");
|
|
39
39
|
|
|
40
|
+
// src/exporters/retry.ts
|
|
41
|
+
var RETRY_STATUSES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
|
|
42
|
+
var TOO_LARGE_STATUS = 413;
|
|
43
|
+
var RETRY_BACKOFF_BASE_S = 1;
|
|
44
|
+
var RETRY_BACKOFF_CAP_S = 30;
|
|
45
|
+
var DEFAULT_RETRY_HORIZON_S = 300;
|
|
46
|
+
var DEFAULT_SHUTDOWN_TIMEOUT_S = 10;
|
|
47
|
+
var RETRY_WARN_AFTER_ATTEMPTS = 3;
|
|
48
|
+
var jitter = { fraction: () => Math.random() };
|
|
49
|
+
function backoffDelayS(retryNumber, fraction) {
|
|
50
|
+
const exponent = Math.min(Math.max(retryNumber, 0), 62);
|
|
51
|
+
const ceiling = Math.min(RETRY_BACKOFF_CAP_S, RETRY_BACKOFF_BASE_S * 2 ** exponent);
|
|
52
|
+
return Math.min(Math.max(fraction, 0), 1) * ceiling;
|
|
53
|
+
}
|
|
54
|
+
var SECONDS = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
|
|
55
|
+
function parseRetryAfterS(value, nowMs = Date.now()) {
|
|
56
|
+
if (value === null || value === void 0) return null;
|
|
57
|
+
const text = String(value).trim();
|
|
58
|
+
if (text === "") return null;
|
|
59
|
+
if (SECONDS.test(text)) {
|
|
60
|
+
const seconds = Number(text);
|
|
61
|
+
return Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
|
|
62
|
+
}
|
|
63
|
+
if (!/[A-Za-z]/.test(text)) return null;
|
|
64
|
+
const when = Date.parse(text);
|
|
65
|
+
if (!Number.isFinite(when)) return null;
|
|
66
|
+
return Math.max(0, (when - nowMs) / 1e3);
|
|
67
|
+
}
|
|
68
|
+
|
|
40
69
|
// src/globals.ts
|
|
41
70
|
var import_node_async_hooks = require("async_hooks");
|
|
42
71
|
var G = globalThis;
|
|
@@ -206,10 +235,24 @@ var VALUE_RULE_PREFIXES = new Set(
|
|
|
206
235
|
);
|
|
207
236
|
|
|
208
237
|
// src/exporters/batch.ts
|
|
238
|
+
var SHUTDOWN_BUDGET_MS = DEFAULT_SHUTDOWN_TIMEOUT_S * 1e3;
|
|
209
239
|
var MAX_TIMEOUT_MS = 2 ** 31 - 1;
|
|
210
240
|
var MAX_BATCH_BYTES = 8 * 1024 * 1024;
|
|
211
241
|
var utf8ByteLength = typeof Buffer !== "undefined" && typeof Buffer.byteLength === "function" ? (s) => Buffer.byteLength(s, "utf8") : (s) => s.length * 3;
|
|
212
242
|
|
|
243
|
+
// src/context/storage.ts
|
|
244
|
+
function storage() {
|
|
245
|
+
return getContextStorage();
|
|
246
|
+
}
|
|
247
|
+
function getContext() {
|
|
248
|
+
return storage().getStore() ?? {};
|
|
249
|
+
}
|
|
250
|
+
function runWithContext(overrides, fn) {
|
|
251
|
+
const parent = getContext();
|
|
252
|
+
const merged = { ...parent, ...overrides };
|
|
253
|
+
return storage().run(merged, fn);
|
|
254
|
+
}
|
|
255
|
+
|
|
213
256
|
// src/ids.ts
|
|
214
257
|
var import_node_crypto = require("crypto");
|
|
215
258
|
|
|
@@ -514,19 +557,6 @@ function classifyError(error) {
|
|
|
514
557
|
return CANONICAL_ERROR_CODES.has(GENERIC_FALLBACK) ? GENERIC_FALLBACK : "";
|
|
515
558
|
}
|
|
516
559
|
|
|
517
|
-
// src/context/storage.ts
|
|
518
|
-
function storage() {
|
|
519
|
-
return getContextStorage();
|
|
520
|
-
}
|
|
521
|
-
function getContext() {
|
|
522
|
-
return storage().getStore() ?? {};
|
|
523
|
-
}
|
|
524
|
-
function runWithContext(overrides, fn) {
|
|
525
|
-
const parent = getContext();
|
|
526
|
-
const merged = { ...parent, ...overrides };
|
|
527
|
-
return storage().run(merged, fn);
|
|
528
|
-
}
|
|
529
|
-
|
|
530
560
|
// src/tracer.ts
|
|
531
561
|
function attachErrorCode(span, error) {
|
|
532
562
|
try {
|
|
@@ -541,7 +571,7 @@ function attachErrorCode(span, error) {
|
|
|
541
571
|
}
|
|
542
572
|
|
|
543
573
|
// src/version.ts
|
|
544
|
-
var SDK_VERSION = "0.
|
|
574
|
+
var SDK_VERSION = "0.11.0";
|
|
545
575
|
|
|
546
576
|
// src/exporters/quota.ts
|
|
547
577
|
var QUOTA_REASON = "monthly_quota_exceeded";
|
|
@@ -649,12 +679,18 @@ var QuotaPause = class {
|
|
|
649
679
|
|
|
650
680
|
// src/exporters/http.ts
|
|
651
681
|
var CAPACITY_REFUSAL_STATUSES = /* @__PURE__ */ new Set([429, 503]);
|
|
652
|
-
var RETRY_AFTER_CEILING_MS = 1e4;
|
|
653
|
-
var REFUSAL_BACKOFF_BASE_MS = 1e3;
|
|
654
|
-
var REFUSAL_NO_HINT = -1;
|
|
655
682
|
var MISROUTE_WARN_WINDOW_MS = 3e5;
|
|
656
683
|
var AUTH_REFUSAL_STATUSES = /* @__PURE__ */ new Set([401, 403]);
|
|
657
684
|
var FAILURE_WARN_WINDOW_MS = 3e5;
|
|
685
|
+
var MAX_TIMER_MS = 2 ** 31 - 1;
|
|
686
|
+
function validHorizonS(value) {
|
|
687
|
+
if (value === void 0) return DEFAULT_RETRY_HORIZON_S;
|
|
688
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
|
|
689
|
+
warn(
|
|
690
|
+
`Risicare: retryHorizonS=${String(value)} is not a finite number of 0 or more; using the default, ${DEFAULT_RETRY_HORIZON_S} s.`
|
|
691
|
+
);
|
|
692
|
+
return DEFAULT_RETRY_HORIZON_S;
|
|
693
|
+
}
|
|
658
694
|
var HttpExporter = class {
|
|
659
695
|
name = "http";
|
|
660
696
|
_endpoint;
|
|
@@ -664,20 +700,28 @@ var HttpExporter = class {
|
|
|
664
700
|
_timeoutMs;
|
|
665
701
|
_maxRetries;
|
|
666
702
|
_compress;
|
|
703
|
+
_retryHorizonS;
|
|
704
|
+
/** Total HTTP requests that this exporter sent (each retry and each half of a split counts). */
|
|
705
|
+
attempts = 0;
|
|
667
706
|
// Circuit breaker (CON-201 / B-16). Threshold and cooldown sized for
|
|
668
|
-
// a "30-second prod restart" scenario
|
|
669
|
-
//
|
|
670
|
-
// cooldown gives the gateway time to come back before the half-open
|
|
671
|
-
// probe fires. Lower numbers (was 3@30s) tripped on transient blips.
|
|
707
|
+
// a "30-second prod restart" scenario. Since W2-1c only answers that a retry
|
|
708
|
+
// cannot change move it (see the block above).
|
|
672
709
|
_consecutiveFailures = 0;
|
|
673
710
|
_lastMisrouteWarnAt = -Infinity;
|
|
674
711
|
_lastFailureWarnAt = -Infinity;
|
|
712
|
+
_lastExhaustedWarnAt = -Infinity;
|
|
713
|
+
/** "log once" (W2-1c): the WARNING for a `too_large` drop is written one time. */
|
|
714
|
+
_tooLargeWarned = false;
|
|
675
715
|
/**
|
|
676
716
|
* The kind of failure that the last outage WARNING named ('auth', 'http', 'none',
|
|
677
717
|
* 'unknown'). `null` after a success. A new kind in the same outage writes one more
|
|
678
718
|
* WARNING, also inside the window.
|
|
679
719
|
*/
|
|
680
720
|
_failureKind = null;
|
|
721
|
+
/** `endRetries()` was called, and `shutdown()` has not run since. */
|
|
722
|
+
_noWait = false;
|
|
723
|
+
/** The wake-ups of the backoff waits in progress. `endRetries()` calls each one. */
|
|
724
|
+
_waits = /* @__PURE__ */ new Set();
|
|
681
725
|
/** The endpoint as a log line may show it: no userinfo, no query (see `safeUrl`). */
|
|
682
726
|
_safeEndpoint;
|
|
683
727
|
/** The endpoint holds userinfo. `fetch` refuses such a URL. */
|
|
@@ -688,6 +732,8 @@ var HttpExporter = class {
|
|
|
688
732
|
_acks = /* @__PURE__ */ new WeakMap();
|
|
689
733
|
/** The requests that each finished call sent, by its batch. Read once by `takeAttempts(spans)`. */
|
|
690
734
|
_requests = /* @__PURE__ */ new WeakMap();
|
|
735
|
+
/** The drop reason of each failed call, by its batch. Read once by `takeDropReason(spans)`. */
|
|
736
|
+
_dropReasons = /* @__PURE__ */ new WeakMap();
|
|
691
737
|
/** The batch processor dropped span(s) from a full queue in the quota pause (S-1). */
|
|
692
738
|
noteQuotaDrop(count = 1) {
|
|
693
739
|
this._quota.addDrops(count);
|
|
@@ -712,8 +758,16 @@ var HttpExporter = class {
|
|
|
712
758
|
this._projectId = options.projectId;
|
|
713
759
|
this._environment = options.environment;
|
|
714
760
|
this._timeoutMs = options.timeoutMs ?? 5e3;
|
|
715
|
-
|
|
761
|
+
let maxRetries = options.maxRetries;
|
|
762
|
+
if (maxRetries !== void 0 && !(Number.isInteger(maxRetries) && maxRetries >= 1)) {
|
|
763
|
+
warn(
|
|
764
|
+
`Risicare: maxRetries=${String(maxRetries)} is not a whole number of 1 or more; the exporter uses no count limit (the retry horizon is the limit).`
|
|
765
|
+
);
|
|
766
|
+
maxRetries = void 0;
|
|
767
|
+
}
|
|
768
|
+
this._maxRetries = maxRetries;
|
|
716
769
|
this._compress = options.compress ?? false;
|
|
770
|
+
this._retryHorizonS = validHorizonS(options.retryHorizonS);
|
|
717
771
|
this._safeEndpoint = safeUrl(this._endpoint);
|
|
718
772
|
this._endpointHasCredentials = hasCredentials(this._endpoint);
|
|
719
773
|
}
|
|
@@ -735,16 +789,18 @@ var HttpExporter = class {
|
|
|
735
789
|
[/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
|
|
736
790
|
return `HttpExporter { endpoint: '${this._safeEndpoint}' }`;
|
|
737
791
|
}
|
|
738
|
-
/**
|
|
792
|
+
/**
|
|
793
|
+
* The most attempts for one request body (`maxRetries`). `undefined` when no count is set:
|
|
794
|
+
* the retry horizon is the limit.
|
|
795
|
+
*/
|
|
739
796
|
get attemptsPerExport() {
|
|
740
797
|
return this._maxRetries;
|
|
741
798
|
}
|
|
742
799
|
/**
|
|
743
800
|
* Returns the number of HTTP requests that the `export()` call for `spans` sent, and clears
|
|
744
|
-
* it.
|
|
745
|
-
*
|
|
746
|
-
*
|
|
747
|
-
* kept per batch, as `takeAck` keeps the ack.
|
|
801
|
+
* it. Each retry and each half of a 413 split counts one; a call that the open circuit
|
|
802
|
+
* breaker or the quota pause stops sends none. Returns `null` when no call is on record for
|
|
803
|
+
* `spans`. The answer is kept per batch, as `takeAck` keeps the ack.
|
|
748
804
|
*/
|
|
749
805
|
takeAttempts(spans) {
|
|
750
806
|
const n = this._requests.get(spans) ?? null;
|
|
@@ -761,6 +817,12 @@ var HttpExporter = class {
|
|
|
761
817
|
this._acks.delete(spans);
|
|
762
818
|
return ack;
|
|
763
819
|
}
|
|
820
|
+
/** Returns why the `export()` call for `spans` failed (W2-1c), and clears it. See `SpanExporter`. */
|
|
821
|
+
takeDropReason(spans) {
|
|
822
|
+
const reason = this._dropReasons.get(spans) ?? null;
|
|
823
|
+
this._dropReasons.delete(spans);
|
|
824
|
+
return reason;
|
|
825
|
+
}
|
|
764
826
|
/** Returns the spans of `spans` that could not be serialized and were not sent, and clears them. */
|
|
765
827
|
takeUnserializable(spans) {
|
|
766
828
|
const bad = this._unserializable.get(spans) ?? [];
|
|
@@ -769,7 +831,7 @@ var HttpExporter = class {
|
|
|
769
831
|
}
|
|
770
832
|
/**
|
|
771
833
|
* Cancel the `export()` call for `spans` (all calls when `spans` is not given): its
|
|
772
|
-
* request, its retries and its backoff
|
|
834
|
+
* request, its retries and its backoff wait. The batch processor calls it when
|
|
773
835
|
* `shutdown()` abandons the batch, so that the call does not keep the process alive. The
|
|
774
836
|
* cancelled call returns FAILURE, changes no counter (the circuit breaker included) and
|
|
775
837
|
* logs no WARNING.
|
|
@@ -779,24 +841,39 @@ var HttpExporter = class {
|
|
|
779
841
|
else for (const call of this._calls.values()) call.cancel.abort();
|
|
780
842
|
}
|
|
781
843
|
/**
|
|
782
|
-
*
|
|
844
|
+
* End the retry waits of this exporter until its next `shutdown()`. The batch processor
|
|
845
|
+
* calls it when `shutdown()` starts. A call that waits for a retry makes one more attempt
|
|
846
|
+
* at once. An attempt that fails after this is not retried: the call fails with the drop
|
|
847
|
+
* reason `shutdown_residue`. So a drain against a dead endpoint costs one attempt for each
|
|
848
|
+
* batch, not the shutdown budget.
|
|
849
|
+
*/
|
|
850
|
+
endRetries() {
|
|
851
|
+
this._noWait = true;
|
|
852
|
+
for (const wake of [...this._waits]) wake();
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Send one batch, with the W2-1c retry policy (see the block above the class).
|
|
856
|
+
*
|
|
857
|
+
* Resolves SUCCESS when the backend accounted for the batch: `takeAck` has the counts (and
|
|
858
|
+
* the spans that a 413 split dropped, in `dropped`). Resolves FAILURE when no span was
|
|
859
|
+
* accounted for: `takeDropReason` then says why, and the batch processor drops the batch
|
|
860
|
+
* with that reason.
|
|
783
861
|
*
|
|
784
862
|
* ⚠ NO CALLER BUDGET. `flush(timeoutMs)` only stops WAITING at its deadline; it never
|
|
785
|
-
* cuts a request that is already sent.
|
|
786
|
-
* healthy, slow server look like a failure: the span used a retry, the breaker moved, and
|
|
787
|
-
* the server stored the same spans again. Only `abort()` stops the call, and only
|
|
863
|
+
* cuts a request that is already sent. Only `abort()` stops the call, and only
|
|
788
864
|
* `shutdown()` calls it.
|
|
789
865
|
*/
|
|
790
866
|
async export(spans) {
|
|
791
867
|
if (spans.length === 0) return "success" /* SUCCESS */;
|
|
792
868
|
const call = {
|
|
793
869
|
cancel: new AbortController(),
|
|
794
|
-
|
|
870
|
+
retryAfterS: null,
|
|
795
871
|
misroute: null,
|
|
796
872
|
noRetry: false,
|
|
797
873
|
status: null,
|
|
798
874
|
error: null,
|
|
799
|
-
attempts: 0
|
|
875
|
+
attempts: 0,
|
|
876
|
+
ack: null
|
|
800
877
|
};
|
|
801
878
|
this._calls.set(spans, call);
|
|
802
879
|
try {
|
|
@@ -833,83 +910,236 @@ var HttpExporter = class {
|
|
|
833
910
|
);
|
|
834
911
|
}
|
|
835
912
|
if (parts.length === 0) return "success" /* SUCCESS */;
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
913
|
+
if (this._consecutiveFailures >= this._circuitBreakerThreshold && Date.now() < this._circuitOpenUntil) {
|
|
914
|
+
this._dropReasons.set(spans, "transport_refused");
|
|
915
|
+
return "failure" /* FAILURE */;
|
|
916
|
+
}
|
|
917
|
+
const out = { delivered: [], dropped: {}, failed: [], cancelled: false };
|
|
918
|
+
const start = performance.now();
|
|
919
|
+
await this._deliver(parts, start, call, out, true);
|
|
920
|
+
if (cancelled.aborted || out.cancelled) {
|
|
921
|
+
this._acks.delete(spans);
|
|
922
|
+
return "failure" /* FAILURE */;
|
|
923
|
+
}
|
|
924
|
+
return this._finish(spans, out, call.attempts);
|
|
925
|
+
}
|
|
926
|
+
/** The request body for `parts` (the serialized spans of one part of a batch). */
|
|
927
|
+
_payload(parts) {
|
|
928
|
+
return `{"spans":[${parts.join(",")}]` + (this._projectId ? `,"projectId":${JSON.stringify(this._projectId)}` : "") + (this._environment ? `,"environment":${JSON.stringify(this._environment)}` : "") + "}";
|
|
929
|
+
}
|
|
930
|
+
/**
|
|
931
|
+
* Send one part of a batch with the retry policy, and record its outcome in `out`.
|
|
932
|
+
* `maySplit` is true only for the whole batch: a 413 splits it in two one time. A half
|
|
933
|
+
* that gets 413 again is dropped as `too_large`.
|
|
934
|
+
*/
|
|
935
|
+
async _deliver(parts, start, call, out, maySplit) {
|
|
936
|
+
const n = parts.length;
|
|
937
|
+
if (!maySplit && this._quota.paused()) {
|
|
938
|
+
out.delivered.push({ n, ack: { accepted: 0, rejected: 0, quotaDropped: n } });
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
const result = await this._sendWithRetries(this._payload(parts), n, start, call);
|
|
942
|
+
if (result.kind === "delivered") {
|
|
943
|
+
out.delivered.push({ n, ack: result.ack });
|
|
944
|
+
} else if (result.kind === "too_large") {
|
|
945
|
+
if (maySplit && n > 1) {
|
|
946
|
+
const mid = Math.ceil(n / 2);
|
|
947
|
+
debug(`Ingest answered HTTP 413 for ${n} span(s): sending two halves of ${mid} and ${n - mid}`);
|
|
948
|
+
await this._deliver(parts.slice(0, mid), start, call, out, false);
|
|
949
|
+
if (!out.cancelled) await this._deliver(parts.slice(mid), start, call, out, false);
|
|
950
|
+
} else {
|
|
951
|
+
out.dropped.too_large = (out.dropped.too_large ?? 0) + n;
|
|
842
952
|
}
|
|
843
|
-
|
|
953
|
+
} else if (result.kind === "cancelled") {
|
|
954
|
+
out.cancelled = true;
|
|
955
|
+
} else {
|
|
956
|
+
out.failed.push({ n, reason: result.kind, answer: result.answer });
|
|
844
957
|
}
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
958
|
+
}
|
|
959
|
+
/** Send one request body until it is delivered or the policy stops. */
|
|
960
|
+
async _sendWithRetries(body, spansSent, start, call) {
|
|
961
|
+
const cancelled = call.cancel.signal;
|
|
962
|
+
let retryNumber = 0;
|
|
963
|
+
let attemptsHere = 0;
|
|
964
|
+
for (; ; ) {
|
|
965
|
+
if (cancelled.aborted) return { kind: "cancelled" };
|
|
966
|
+
call.retryAfterS = null;
|
|
848
967
|
call.misroute = null;
|
|
849
968
|
call.noRetry = false;
|
|
850
969
|
call.status = null;
|
|
851
970
|
call.error = null;
|
|
971
|
+
call.ack = null;
|
|
852
972
|
call.attempts++;
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
973
|
+
attemptsHere++;
|
|
974
|
+
this.attempts++;
|
|
975
|
+
const result = await this._sendRequest(body, spansSent, call);
|
|
976
|
+
if (cancelled.aborted) return { kind: "cancelled" };
|
|
977
|
+
if (result === "success" /* SUCCESS */) return { kind: "delivered", ack: call.ack };
|
|
978
|
+
if (call.status === TOO_LARGE_STATUS) return { kind: "too_large" };
|
|
979
|
+
const answer = {
|
|
980
|
+
status: call.status,
|
|
981
|
+
error: call.error,
|
|
982
|
+
misroute: call.misroute,
|
|
983
|
+
zeroAck: call.noRetry,
|
|
984
|
+
sent: spansSent
|
|
985
|
+
};
|
|
986
|
+
if (!this._retryable(call)) return { kind: "transport_refused", answer };
|
|
987
|
+
if (attemptsHere === RETRY_WARN_AFTER_ATTEMPTS) this._warnOutage(answer, true);
|
|
988
|
+
if (this._noWait) return { kind: "shutdown_residue", answer };
|
|
989
|
+
if (this._maxRetries !== void 0 && attemptsHere >= this._maxRetries) {
|
|
990
|
+
return { kind: "retry_exhausted", answer };
|
|
857
991
|
}
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
992
|
+
const remainingS = this._retryHorizonS - (performance.now() - start) / 1e3;
|
|
993
|
+
if (remainingS <= 0) return { kind: "retry_exhausted", answer };
|
|
994
|
+
const hint = call.retryAfterS;
|
|
995
|
+
const waitS = hint !== null && hint > 0 ? hint : backoffDelayS(retryNumber, jitter.fraction());
|
|
996
|
+
const delayS = Math.min(waitS, remainingS);
|
|
997
|
+
retryNumber++;
|
|
998
|
+
debug(
|
|
999
|
+
`Ingest attempt ${attemptsHere} failed (${answer.status ?? answer.error ?? "unknown"}); next attempt in ${delayS.toFixed(2)} s (${Math.round(remainingS)} s of the horizon left)`
|
|
1000
|
+
);
|
|
1001
|
+
await this._retryWait(delayS * 1e3, cancelled);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
/** True when the answer of the last attempt of `call` can change on a retry. */
|
|
1005
|
+
_retryable(call) {
|
|
1006
|
+
if (call.misroute !== null || call.noRetry) return false;
|
|
1007
|
+
if (call.status !== null) return RETRY_STATUSES.has(call.status);
|
|
1008
|
+
if (this._endpointHasCredentials) return false;
|
|
1009
|
+
return call.error !== null;
|
|
1010
|
+
}
|
|
1011
|
+
/**
|
|
1012
|
+
* Wait `ms` before the next attempt. `abort()` and `endRetries()` end the wait at once. The
|
|
1013
|
+
* timer does not keep the process alive: when nothing else runs, `beforeExit` starts the
|
|
1014
|
+
* SDK's `shutdown()`, which ends the wait.
|
|
1015
|
+
*/
|
|
1016
|
+
_retryWait(ms, signal) {
|
|
1017
|
+
return new Promise((resolve) => {
|
|
1018
|
+
if (signal.aborted || this._noWait || ms <= 0) return resolve();
|
|
1019
|
+
const done = () => {
|
|
1020
|
+
clearTimeout(timer);
|
|
1021
|
+
signal.removeEventListener("abort", done);
|
|
1022
|
+
this._waits.delete(done);
|
|
1023
|
+
resolve();
|
|
1024
|
+
};
|
|
1025
|
+
const timer = setTimeout(done, Math.min(ms, MAX_TIMER_MS));
|
|
1026
|
+
timer.unref?.();
|
|
1027
|
+
signal.addEventListener("abort", done, { once: true });
|
|
1028
|
+
this._waits.add(done);
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
/** Turn the outcome of the parts of one batch into the result of `export()`. */
|
|
1032
|
+
_finish(spans, out, attempts) {
|
|
1033
|
+
const reasons = new Set(out.failed.map((f) => f.reason));
|
|
1034
|
+
if (out.delivered.length === 0 && Object.keys(out.dropped).length === 0 && reasons.size === 1) {
|
|
1035
|
+
const reason = out.failed[0].reason;
|
|
1036
|
+
const nFailed = out.failed.reduce((sum, f) => sum + f.n, 0);
|
|
1037
|
+
this._dropReasons.set(spans, reason);
|
|
1038
|
+
return this._fail(reason, out.failed[out.failed.length - 1].answer, nFailed, attempts);
|
|
1039
|
+
}
|
|
1040
|
+
let ack;
|
|
1041
|
+
if (out.delivered.length === 1 && Object.keys(out.dropped).length === 0 && out.failed.length === 0) {
|
|
1042
|
+
ack = out.delivered[0].ack;
|
|
1043
|
+
} else {
|
|
1044
|
+
let accepted = 0;
|
|
1045
|
+
let rejected = 0;
|
|
1046
|
+
let quotaDropped = 0;
|
|
1047
|
+
for (const part of out.delivered) {
|
|
1048
|
+
accepted += part.ack !== null ? part.ack.accepted : part.n;
|
|
1049
|
+
rejected += part.ack !== null ? part.ack.rejected : 0;
|
|
1050
|
+
quotaDropped += part.ack?.quotaDropped ?? 0;
|
|
870
1051
|
}
|
|
1052
|
+
const dropped = { ...out.dropped };
|
|
1053
|
+
for (const f of out.failed) dropped[f.reason] = (dropped[f.reason] ?? 0) + f.n;
|
|
1054
|
+
ack = { accepted, rejected };
|
|
1055
|
+
if (quotaDropped > 0) ack.quotaDropped = quotaDropped;
|
|
1056
|
+
if (Object.keys(dropped).length > 0) ack.dropped = dropped;
|
|
871
1057
|
}
|
|
872
|
-
if (
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
1058
|
+
if (ack !== null) this._acks.set(spans, ack);
|
|
1059
|
+
this._consecutiveFailures = 0;
|
|
1060
|
+
this._failureKind = null;
|
|
1061
|
+
const tooLarge = out.dropped.too_large ?? 0;
|
|
1062
|
+
if (tooLarge > 0) this._warnTooLarge(tooLarge);
|
|
1063
|
+
for (const f of out.failed) {
|
|
1064
|
+
if (f.reason === "retry_exhausted") this._warnExhausted(f.n, attempts, f.answer);
|
|
1065
|
+
}
|
|
1066
|
+
return "success" /* SUCCESS */;
|
|
1067
|
+
}
|
|
1068
|
+
/** End a call in which no span was accounted for. Always returns FAILURE. */
|
|
1069
|
+
_fail(reason, answer, nSpans, attempts) {
|
|
1070
|
+
if (reason === "shutdown_residue") {
|
|
1071
|
+
debug(`Export ended by shutdown after ${attempts} attempt(s)`);
|
|
876
1072
|
return "failure" /* FAILURE */;
|
|
877
1073
|
}
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
this.
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
);
|
|
1074
|
+
if (reason === "retry_exhausted") {
|
|
1075
|
+
if (answer.status === null || !CAPACITY_REFUSAL_STATUSES.has(answer.status)) this._stepBreaker();
|
|
1076
|
+
this._warnOutage(answer, true);
|
|
1077
|
+
this._warnExhausted(nSpans, attempts, answer);
|
|
1078
|
+
return "failure" /* FAILURE */;
|
|
884
1079
|
}
|
|
1080
|
+
this._stepBreaker();
|
|
1081
|
+
this._warnOutage(answer, false);
|
|
1082
|
+
return "failure" /* FAILURE */;
|
|
1083
|
+
}
|
|
1084
|
+
/** Count one failed call toward the circuit breaker. */
|
|
1085
|
+
_stepBreaker() {
|
|
885
1086
|
this._consecutiveFailures++;
|
|
886
|
-
if (misroute === null && !call.noRetry) {
|
|
887
|
-
const status = call.status;
|
|
888
|
-
const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : call.error !== null ? "none" : "unknown";
|
|
889
|
-
let write = false;
|
|
890
|
-
if (this._consecutiveFailures === 1 || this._failureKind === null) {
|
|
891
|
-
if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
|
|
892
|
-
this._lastFailureWarnAt = Date.now();
|
|
893
|
-
write = true;
|
|
894
|
-
}
|
|
895
|
-
this._failureKind = kind;
|
|
896
|
-
} else if (kind !== this._failureKind) {
|
|
897
|
-
this._failureKind = kind;
|
|
898
|
-
write = true;
|
|
899
|
-
}
|
|
900
|
-
if (write) this._warnOutage(kind, status, call.error);
|
|
901
|
-
}
|
|
902
1087
|
if (this._consecutiveFailures >= this._circuitBreakerThreshold) {
|
|
903
1088
|
this._circuitOpenUntil = Date.now() + this._circuitBreakerCooldownMs;
|
|
904
1089
|
warn(
|
|
905
1090
|
`HTTP exporter circuit breaker opened after ${this._consecutiveFailures} failures. Cooldown: ${this._circuitBreakerCooldownMs / 1e3}s`
|
|
906
1091
|
);
|
|
907
1092
|
}
|
|
908
|
-
return "failure" /* FAILURE */;
|
|
909
1093
|
}
|
|
910
|
-
/**
|
|
911
|
-
|
|
1094
|
+
/**
|
|
1095
|
+
* The outage WARNING: at the first failure of an outage, and when its kind changes. At most
|
|
1096
|
+
* one in the window for the first failure; a misroute has its own WARNING, at most one in
|
|
1097
|
+
* the window. `retrying` is true for a retryable answer (written after
|
|
1098
|
+
* `RETRY_WARN_AFTER_ATTEMPTS` failed attempts of one batch, or when its retries end before
|
|
1099
|
+
* that), false for an answer that a retry cannot change (one attempt). The WARNING names the last HTTP status, or the error
|
|
1100
|
+
* code when there was no answer. It never names the key or the response body, and names
|
|
1101
|
+
* the API key as a cause only for a 401 or 403. An ack of 0 and 0 has its WARNING from
|
|
1102
|
+
* `checkAck`.
|
|
1103
|
+
*/
|
|
1104
|
+
_warnOutage(answer, retrying) {
|
|
1105
|
+
if (answer.misroute !== null) {
|
|
1106
|
+
if (Date.now() - this._lastMisrouteWarnAt > MISROUTE_WARN_WINDOW_MS) {
|
|
1107
|
+
this._lastMisrouteWarnAt = Date.now();
|
|
1108
|
+
warn(
|
|
1109
|
+
`Risicare: spans are NOT reaching ${this._safeEndpoint}. It answered POST /v1/spans with ${answer.misroute}. That is not the Risicare ingest gateway, so the endpoint is probably wrong (for example the dashboard URL instead of the ingest gateway URL). The SDK does not follow redirects, and this batch is counted as NOT delivered. Set endpoint or RISICARE_ENDPOINT to the ingest gateway URL.`
|
|
1110
|
+
);
|
|
1111
|
+
}
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
if (answer.zeroAck) return;
|
|
1115
|
+
const status = answer.status;
|
|
1116
|
+
const kind = status !== null && AUTH_REFUSAL_STATUSES.has(status) ? "auth" : status !== null ? "http" : answer.error !== null ? "none" : "unknown";
|
|
1117
|
+
let write = false;
|
|
1118
|
+
if (this._failureKind === null || !retrying && this._consecutiveFailures === 1) {
|
|
1119
|
+
if (Date.now() - this._lastFailureWarnAt > FAILURE_WARN_WINDOW_MS) {
|
|
1120
|
+
this._lastFailureWarnAt = Date.now();
|
|
1121
|
+
write = true;
|
|
1122
|
+
}
|
|
1123
|
+
this._failureKind = kind;
|
|
1124
|
+
} else if (kind !== this._failureKind) {
|
|
1125
|
+
this._failureKind = kind;
|
|
1126
|
+
write = true;
|
|
1127
|
+
}
|
|
1128
|
+
if (!write) return;
|
|
912
1129
|
const where = `Risicare: spans are NOT reaching ${this._safeEndpoint}`;
|
|
1130
|
+
if (retrying) {
|
|
1131
|
+
let limit = `${this._retryHorizonS} s from its first attempt (retryHorizonS)`;
|
|
1132
|
+
if (this._maxRetries !== void 0) limit = `${this._maxRetries} attempt(s) (maxRetries) or ${limit}`;
|
|
1133
|
+
const retryText = `The SDK sends each batch again with a growing wait, for up to ${limit}, and then drops it (reason=retry_exhausted).`;
|
|
1134
|
+
if (kind === "http") {
|
|
1135
|
+
warn(`${where}: it answered HTTP ${status}. ${retryText} Set debug: true to see each attempt.`);
|
|
1136
|
+
} else {
|
|
1137
|
+
warn(
|
|
1138
|
+
`${where}: no answer (${answer.error}). The endpoint cannot be reached, or it did not answer in time. ${retryText} Set debug: true to see each attempt.`
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
913
1143
|
const tail = "Tracing continues, but the SDK drops the spans that it cannot send.";
|
|
914
1144
|
if (kind === "auth") {
|
|
915
1145
|
warn(
|
|
@@ -917,15 +1147,11 @@ var HttpExporter = class {
|
|
|
917
1147
|
);
|
|
918
1148
|
} else if (kind === "http") {
|
|
919
1149
|
warn(
|
|
920
|
-
`${where}: it answered HTTP ${status}. The server did not take the batch. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
|
|
1150
|
+
`${where}: it answered HTTP ${status}. The server did not take the batch, and a retry cannot change that answer. Check the endpoint and the server. ${tail} Set debug: true to see each attempt.`
|
|
921
1151
|
);
|
|
922
1152
|
} else if (kind === "none" && this._endpointHasCredentials) {
|
|
923
1153
|
warn(
|
|
924
|
-
`${where}: no answer (${error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
|
|
925
|
-
);
|
|
926
|
-
} else if (kind === "none") {
|
|
927
|
-
warn(
|
|
928
|
-
`${where}: no answer (${error}). The endpoint cannot be reached, or it did not answer in time. Check the endpoint and the network. ${tail} Set debug: true to see each attempt.`
|
|
1154
|
+
`${where}: no answer (${answer.error}). The endpoint URL holds credentials (a user name or a password before the host), and fetch does not send a request to such a URL. Remove the credentials from the endpoint URL. ${tail}`
|
|
929
1155
|
);
|
|
930
1156
|
} else {
|
|
931
1157
|
warn(
|
|
@@ -933,33 +1159,38 @@ var HttpExporter = class {
|
|
|
933
1159
|
);
|
|
934
1160
|
}
|
|
935
1161
|
}
|
|
1162
|
+
/** The WARNING for spans dropped as `retry_exhausted`. At most one in the window. */
|
|
1163
|
+
_warnExhausted(nSpans, attempts, answer) {
|
|
1164
|
+
if (Date.now() - this._lastExhaustedWarnAt <= FAILURE_WARN_WINDOW_MS) return;
|
|
1165
|
+
this._lastExhaustedWarnAt = Date.now();
|
|
1166
|
+
const last = answer.status !== null ? `HTTP ${answer.status}` : `no answer (${answer.error})`;
|
|
1167
|
+
const capacity = answer.status !== null && CAPACITY_REFUSAL_STATUSES.has(answer.status) ? " (over capacity)" : "";
|
|
1168
|
+
const limit = this._maxRetries !== void 0 && attempts >= this._maxRetries ? `the maxRetries limit of ${this._maxRetries} attempt(s)` : `the retry horizon of ${this._retryHorizonS} s (retryHorizonS)`;
|
|
1169
|
+
warn(
|
|
1170
|
+
`Risicare: ${nSpans} span(s) DISCARDED (reason=retry_exhausted): ${this._safeEndpoint} did not take them in ${attempts} attempt(s), and ${limit} is used up. The last answer: ${last}${capacity}. A gateway outage that is longer than the retry horizon loses spans. To wait longer, raise retryHorizonS.`
|
|
1171
|
+
);
|
|
1172
|
+
}
|
|
1173
|
+
/** The WARNING for spans dropped as `too_large`. One time for each exporter. */
|
|
1174
|
+
_warnTooLarge(nSpans) {
|
|
1175
|
+
if (this._tooLargeWarned) return;
|
|
1176
|
+
this._tooLargeWarned = true;
|
|
1177
|
+
warn(
|
|
1178
|
+
`Risicare: ${nSpans} span(s) DISCARDED (reason=too_large): ${this._safeEndpoint} answered HTTP 413 (the request is too large) for a batch, and again for a half of it after one split. The SDK does not send these spans again. Make the span attributes smaller. This WARNING is written one time; droppedSpansByReason.too_large has the count.`
|
|
1179
|
+
);
|
|
1180
|
+
}
|
|
936
1181
|
/** Record a MISROUTE for the retry loop and the breaker. Always returns FAILURE. */
|
|
937
1182
|
_noteMisroute(call, answer) {
|
|
938
1183
|
call.misroute = this._scrub(answer, 300);
|
|
939
1184
|
debug(`Ingest misroute: ${call.misroute}`);
|
|
940
1185
|
return "failure" /* FAILURE */;
|
|
941
1186
|
}
|
|
942
|
-
/**
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
* parse.
|
|
948
|
-
*/
|
|
949
|
-
_noteRefusal(call, status, retryAfterRaw) {
|
|
950
|
-
if (!CAPACITY_REFUSAL_STATUSES.has(status)) return false;
|
|
951
|
-
let delay = REFUSAL_NO_HINT;
|
|
952
|
-
if (retryAfterRaw) {
|
|
953
|
-
const secs = Number(retryAfterRaw);
|
|
954
|
-
if (Number.isFinite(secs) && secs >= 0) {
|
|
955
|
-
delay = Math.min(secs * 1e3, RETRY_AFTER_CEILING_MS);
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
call.refusalAfterMs = delay;
|
|
959
|
-
debug(`Ingest refused for capacity: HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
|
|
960
|
-
return true;
|
|
1187
|
+
/** Record the `Retry-After` wait of a retryable answer for the retry loop. */
|
|
1188
|
+
_noteRetryAfter(call, status, retryAfterRaw) {
|
|
1189
|
+
if (!RETRY_STATUSES.has(status)) return;
|
|
1190
|
+
call.retryAfterS = parseRetryAfterS(retryAfterRaw);
|
|
1191
|
+
debug(`Ingest answered HTTP ${status} (Retry-After=${this._scrub(retryAfterRaw)})`);
|
|
961
1192
|
}
|
|
962
|
-
async _sendRequest(body, spansSent, call
|
|
1193
|
+
async _sendRequest(body, spansSent, call) {
|
|
963
1194
|
const cancelled = call.cancel.signal;
|
|
964
1195
|
const timeoutMs = this._timeoutMs;
|
|
965
1196
|
const url = `${this._endpoint}/v1/spans`;
|
|
@@ -1027,14 +1258,15 @@ var HttpExporter = class {
|
|
|
1027
1258
|
} else if (ack.result === "success" /* SUCCESS */) {
|
|
1028
1259
|
this._quota.accepted(`the ingest endpoint ${this._safeEndpoint}`);
|
|
1029
1260
|
}
|
|
1030
|
-
if (ack.result === "success" /* SUCCESS */)
|
|
1261
|
+
if (ack.result === "success" /* SUCCESS */) call.ack = ack.counts;
|
|
1031
1262
|
if (ack.final) call.noRetry = true;
|
|
1032
1263
|
return ack.result;
|
|
1033
|
-
}
|
|
1264
|
+
}
|
|
1265
|
+
this._noteRetryAfter(call, response.status, response.headers?.get?.("Retry-After") ?? null);
|
|
1266
|
+
if (response.status === 408 || response.status === 504) {
|
|
1034
1267
|
debug(`HTTP export timeout: ${response.status}`);
|
|
1035
1268
|
return "timeout" /* TIMEOUT */;
|
|
1036
1269
|
}
|
|
1037
|
-
this._noteRefusal(call, response.status, response.headers.get("Retry-After"));
|
|
1038
1270
|
debug(`HTTP export failed: ${response.status}`);
|
|
1039
1271
|
return "failure" /* FAILURE */;
|
|
1040
1272
|
} catch (e) {
|
|
@@ -1048,20 +1280,9 @@ var HttpExporter = class {
|
|
|
1048
1280
|
}
|
|
1049
1281
|
}
|
|
1050
1282
|
shutdown() {
|
|
1283
|
+
this._noWait = false;
|
|
1051
1284
|
}
|
|
1052
1285
|
};
|
|
1053
|
-
function sleep(ms, signal) {
|
|
1054
|
-
return new Promise((resolve) => {
|
|
1055
|
-
if (signal?.aborted) return resolve();
|
|
1056
|
-
const done = () => {
|
|
1057
|
-
clearTimeout(timer);
|
|
1058
|
-
signal?.removeEventListener("abort", done);
|
|
1059
|
-
resolve();
|
|
1060
|
-
};
|
|
1061
|
-
const timer = setTimeout(done, ms);
|
|
1062
|
-
signal?.addEventListener("abort", done, { once: true });
|
|
1063
|
-
});
|
|
1064
|
-
}
|
|
1065
1286
|
function asCount(v) {
|
|
1066
1287
|
return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : void 0;
|
|
1067
1288
|
}
|