typekro 0.28.0 → 0.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/core/deployment/kro-readiness.d.ts.map +1 -1
- package/dist/core/deployment/kro-readiness.js +30 -5
- package/dist/core/deployment/kro-readiness.js.map +1 -1
- package/dist/core/deployment/poll-timeout.d.ts +45 -0
- package/dist/core/deployment/poll-timeout.d.ts.map +1 -0
- package/dist/core/deployment/poll-timeout.js +67 -0
- package/dist/core/deployment/poll-timeout.js.map +1 -0
- package/dist/core/deployment/readiness.d.ts.map +1 -1
- package/dist/core/deployment/readiness.js +30 -21
- package/dist/core/deployment/readiness.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kro-readiness.d.ts","sourceRoot":"","sources":["../../../src/core/deployment/kro-readiness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,KAAK,GAAG,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"kro-readiness.d.ts","sourceRoot":"","sources":["../../../src/core/deployment/kro-readiness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,KAAK,GAAG,MAAM,yBAAyB,CAAC;AAWpD,kDAAkD;AAClD,MAAM,WAAW,mBAAmB;IAClC,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IAErB,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAEhB,uEAAuE;IACvE,MAAM,EAAE,GAAG,CAAC,mBAAmB,CAAC;IAEhC,+DAA+D;IAC/D,gBAAgB,EAAE,GAAG,CAAC,gBAAgB,CAAC;IAEvC,yDAAyD;IACzD,SAAS,EAAE,MAAM,CAAC;IAElB,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IAEnB,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwPzF"}
|
|
@@ -9,9 +9,10 @@
|
|
|
9
9
|
* - Kro conditions (InstanceSynced for v0.3.x, Ready for v0.8.x)
|
|
10
10
|
* - Custom status fields populated from the RGD status schema
|
|
11
11
|
*/
|
|
12
|
-
import { DEFAULT_FAST_POLL_INTERVAL, DEFAULT_POLL_INTERVAL } from '../config/defaults.js';
|
|
12
|
+
import { DEFAULT_FAST_POLL_INTERVAL, DEFAULT_HTTP_READ_TIMEOUT, DEFAULT_POLL_INTERVAL, } from '../config/defaults.js';
|
|
13
13
|
import { CRDInstanceError, DeploymentTimeoutError, ensureError } from '../errors.js';
|
|
14
14
|
import { getComponentLogger } from '../logging/index.js';
|
|
15
|
+
import { callWithTimeout, perCallTimeout, PollTimeoutError } from './poll-timeout.js';
|
|
15
16
|
/**
|
|
16
17
|
* Wait for a Kro custom resource instance to become ready.
|
|
17
18
|
*
|
|
@@ -31,14 +32,20 @@ export async function waitForKroInstanceReady(options) {
|
|
|
31
32
|
const startTime = Date.now();
|
|
32
33
|
while (Date.now() - startTime < timeout) {
|
|
33
34
|
try {
|
|
34
|
-
|
|
35
|
+
// Bound the read so a wedged/expired kubeconfig exec credential rejects (and is re-thrown below)
|
|
36
|
+
// instead of hanging the poll forever — see poll-timeout.ts. A ≤0 budget means the deadline is
|
|
37
|
+
// spent: break to the overall DeploymentTimeoutError below (NOT a per-call PollTimeoutError).
|
|
38
|
+
const readTimeout = perCallTimeout(timeout - (Date.now() - startTime), DEFAULT_HTTP_READ_TIMEOUT);
|
|
39
|
+
if (readTimeout <= 0)
|
|
40
|
+
break;
|
|
41
|
+
const response = await callWithTimeout(() => k8sApi.read({
|
|
35
42
|
apiVersion,
|
|
36
43
|
kind,
|
|
37
44
|
metadata: {
|
|
38
45
|
name: instanceName,
|
|
39
46
|
namespace,
|
|
40
47
|
},
|
|
41
|
-
});
|
|
48
|
+
}), readTimeout, `read ${kind}/${instanceName}`);
|
|
42
49
|
// In the new API, methods return objects directly (no .body wrapper)
|
|
43
50
|
const instance = response;
|
|
44
51
|
// Kro-specific readiness logic
|
|
@@ -68,12 +75,17 @@ export async function waitForKroInstanceReady(options) {
|
|
|
68
75
|
let expectedCustomStatusFields = false;
|
|
69
76
|
let expectedStatusKeys = [];
|
|
70
77
|
try {
|
|
71
|
-
const
|
|
78
|
+
const rgdReadTimeout = perCallTimeout(timeout - (Date.now() - startTime), DEFAULT_HTTP_READ_TIMEOUT);
|
|
79
|
+
// Deadline spent mid-iteration: break to the overall DeploymentTimeoutError rather than starting
|
|
80
|
+
// a doomed read (which would surface a misleading per-call PollTimeoutError).
|
|
81
|
+
if (rgdReadTimeout <= 0)
|
|
82
|
+
break;
|
|
83
|
+
const rgdResponse = await callWithTimeout(() => customObjectsApi.getClusterCustomObject({
|
|
72
84
|
group: 'kro.run',
|
|
73
85
|
version: 'v1alpha1',
|
|
74
86
|
plural: 'resourcegraphdefinitions',
|
|
75
87
|
name: rgdName,
|
|
76
|
-
});
|
|
88
|
+
}), rgdReadTimeout, `read ResourceGraphDefinition/${rgdName}`);
|
|
77
89
|
const rgd = rgdResponse;
|
|
78
90
|
const rgdStatusSchema = rgd.spec?.schema?.status ?? {};
|
|
79
91
|
expectedStatusKeys = Object.keys(rgdStatusSchema).filter((key) => !basicKroFields.includes(key));
|
|
@@ -85,6 +97,13 @@ export async function waitForKroInstanceReady(options) {
|
|
|
85
97
|
});
|
|
86
98
|
}
|
|
87
99
|
catch (error) {
|
|
100
|
+
// A per-call TIMEOUT (wedged/expired credential) is NOT a fetchable-RGD failure — do not fall
|
|
101
|
+
// through to the permissive path, which would let an ACTIVE/synced instance be declared ready
|
|
102
|
+
// WITHOUT validating expected status fields (and after the deadline). Surface it so the outer
|
|
103
|
+
// catch re-throws it (fail fast).
|
|
104
|
+
if (error instanceof PollTimeoutError) {
|
|
105
|
+
throw error;
|
|
106
|
+
}
|
|
88
107
|
readinessLogger.warn('Could not fetch ResourceGraphDefinition for status schema check', {
|
|
89
108
|
rgdName,
|
|
90
109
|
error: ensureError(error).message,
|
|
@@ -127,6 +146,12 @@ export async function waitForKroInstanceReady(options) {
|
|
|
127
146
|
projectedStatusIsCurrent &&
|
|
128
147
|
(!hasReadyField || statusReadyField === true);
|
|
129
148
|
if (isReady) {
|
|
149
|
+
// Re-check the deadline: an API call earlier in THIS iteration may have consumed the remaining
|
|
150
|
+
// budget (e.g. a slow/bounded read), so an "isReady" reached past the deadline must still time
|
|
151
|
+
// out rather than silently succeed late.
|
|
152
|
+
if (Date.now() - startTime >= timeout) {
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
130
155
|
readinessLogger.info('Kro instance is ready', {
|
|
131
156
|
instanceName,
|
|
132
157
|
hasCustomStatusFields,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kro-readiness.js","sourceRoot":"","sources":["../../../src/core/deployment/kro-readiness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,
|
|
1
|
+
{"version":3,"file":"kro-readiness.js","sourceRoot":"","sources":["../../../src/core/deployment/kro-readiness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAyCtF;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAA4B;IACxE,MAAM,EACJ,YAAY,EACZ,OAAO,EACP,MAAM,EACN,gBAAgB,EAChB,SAAS,EACT,UAAU,EACV,IAAI,EACJ,OAAO,EACP,YAAY,GAAG,0BAA0B,EACzC,cAAc,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACnD,MAAM,eAAe,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,iGAAiG;YACjG,+FAA+F;YAC/F,8FAA8F;YAC9F,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAClG,IAAI,WAAW,IAAI,CAAC;gBAAE,MAAM;YAC5B,MAAM,QAAQ,GAAG,MAAM,eAAe,CACpC,GAAG,EAAE,CACH,MAAM,CAAC,IAAI,CAAC;gBACV,UAAU;gBACV,IAAI;gBACJ,QAAQ,EAAE;oBACR,IAAI,EAAE,YAAY;oBAClB,SAAS;iBACV;aACF,CAAC,EACJ,WAAW,EACX,QAAQ,IAAI,IAAI,YAAY,EAAE,CAC/B,CAAC;YAEF,qEAAqE;YACrE,MAAM,QAAQ,GAAG,QAehB,CAAC;YAEF,+BAA+B;YAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC/B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,eAAe,CAAC,KAAK,CAAC,yCAAyC,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;gBACnF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC;gBAC3E,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;YAC3C,yEAAyE;YACzE,MAAM,eAAe,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,gBAAgB,CAAC,CAAC;YAC5E,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;YAClE,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;YACjD,MAAM,kBAAkB,GAAG,CAAC,SAAkD,EAAE,EAAE,CAChF,SAAS,EAAE,MAAM,KAAK,MAAM;gBAC5B,CAAC,UAAU,KAAK,SAAS;oBACvB,SAAS,CAAC,kBAAkB,KAAK,SAAS;oBAC1C,SAAS,CAAC,kBAAkB,IAAI,UAAU,CAAC,CAAC;YAEhD,6EAA6E;YAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,cAAc,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YAC/C,MAAM,qBAAqB,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAEtF,MAAM,QAAQ,GAAG,KAAK,KAAK,QAAQ,CAAC;YACpC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,eAAe,CAAC,IAAI,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAE3F,kFAAkF;YAClF,IAAI,0BAA0B,GAAG,KAAK,CAAC;YACvC,IAAI,kBAAkB,GAAa,EAAE,CAAC;YACtC,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,cAAc,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,EAAE,yBAAyB,CAAC,CAAC;gBACrG,iGAAiG;gBACjG,8EAA8E;gBAC9E,IAAI,cAAc,IAAI,CAAC;oBAAE,MAAM;gBAC/B,MAAM,WAAW,GAAG,MAAM,eAAe,CACvC,GAAG,EAAE,CACH,gBAAgB,CAAC,sBAAsB,CAAC;oBACtC,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,UAAU;oBACnB,MAAM,EAAE,0BAA0B;oBAClC,IAAI,EAAE,OAAO;iBACd,CAAC,EACJ,cAAc,EACd,gCAAgC,OAAO,EAAE,CAC1C,CAAC;gBACF,MAAM,GAAG,GAAG,WAA0B,CAAC;gBACvC,MAAM,eAAe,GAAG,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;gBACvD,kBAAkB,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM,CACtD,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CACvC,CAAC;gBACF,0BAA0B,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;gBAE3D,eAAe,CAAC,KAAK,CAAC,6CAA6C,EAAE;oBACnE,OAAO;oBACP,aAAa,EAAE,kBAAkB;oBACjC,0BAA0B;iBAC3B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,8FAA8F;gBAC9F,8FAA8F;gBAC9F,8FAA8F;gBAC9F,kCAAkC;gBAClC,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;oBACtC,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,eAAe,CAAC,IAAI,CAAC,iEAAiE,EAAE;oBACtF,OAAO;oBACP,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,OAAO;iBAClC,CAAC,CAAC;gBACH,gGAAgG;gBAChG,0BAA0B,GAAG,KAAK,CAAC;YACrC,CAAC;YAED,eAAe,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBACjD,YAAY;gBACZ,KAAK;gBACL,QAAQ;gBACR,QAAQ;gBACR,qBAAqB;gBACrB,0BAA0B;gBAC1B,kBAAkB;gBAClB,UAAU;gBACV,UAAU;gBACV,wBAAwB,EAAE,eAAe,EAAE,kBAAkB;gBAC7D,uBAAuB,EAAE,cAAc,EAAE,kBAAkB;aAC5D,CAAC,CAAC;YAEH,2EAA2E;YAC3E,4EAA4E;YAC5E,wDAAwD;YACxD,MAAM,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC;YACtC,MAAM,aAAa,GAAG,OAAO,gBAAgB,KAAK,SAAS,CAAC;YAC5D,MAAM,uBAAuB,GAC3B,kBAAkB,CAAC,MAAM,KAAK,CAAC;gBAC/B,kBAAkB,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;YAChE,oEAAoE;YACpE,0EAA0E;YAC1E,sEAAsE;YACtE,qEAAqE;YACrE,2DAA2D;YAC3D,MAAM,2BAA2B,GAAG,MAAM,CAAC,kBAAkB,CAAC;YAC9D,MAAM,wBAAwB,GAC5B,2BAA2B,KAAK,SAAS;gBACzC,UAAU,KAAK,SAAS;gBACxB,2BAA2B,IAAI,UAAU,CAAC;YAC5C,MAAM,OAAO,GACX,QAAQ;gBACR,QAAQ;gBACR,uBAAuB;gBACvB,wBAAwB;gBACxB,CAAC,CAAC,aAAa,IAAI,gBAAgB,KAAK,IAAI,CAAC,CAAC;YAEhD,IAAI,OAAO,EAAE,CAAC;gBACZ,+FAA+F;gBAC/F,+FAA+F;gBAC/F,yCAAyC;gBACzC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,IAAI,OAAO,EAAE,CAAC;oBACtC,MAAM;gBACR,CAAC;gBACD,eAAe,CAAC,IAAI,CAAC,uBAAuB,EAAE;oBAC5C,YAAY;oBACZ,qBAAqB;oBACrB,0BAA0B;oBAC1B,kBAAkB;iBACnB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,2EAA2E;YAC3E,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;gBAC5C,MAAM,eAAe,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC;gBACrE,MAAM,YAAY,GAAG,eAAe,EAAE,OAAO,IAAI,eAAe,CAAC;gBACjE,MAAM,IAAI,gBAAgB,CACxB,yCAAyC,KAAK,MAAM,YAAY,EAAE,EAClE,UAAU,EACV,IAAI,EACJ,YAAY,EACZ,UAAU,CACX,CAAC;YACJ,CAAC;YAED,eAAe,CAAC,KAAK,CAAC,gDAAgD,EAAE;gBACtE,YAAY;gBACZ,KAAK;gBACL,QAAQ;gBACR,qBAAqB;gBACrB,kBAAkB;gBAClB,2BAA2B;gBAC3B,wBAAwB;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,kCAAkC;YAClC,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;gBACtC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,GAAG,KAKhB,CAAC;YACF,MAAM,SAAS,GACb,QAAQ,CAAC,UAAU;gBACnB,QAAQ,CAAC,IAAI,EAAE,IAAI;gBACnB,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAClE,MAAM,UAAU,GACd,SAAS,KAAK,GAAG;gBACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,KAAK,UAAU;gBACpC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,KAAK,IAAI;gBACjD,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC;YAClD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC;YACd,CAAC;YACD,2CAA2C;YAC3C,eAAe,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,6BAA6B;QAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACvC,MAAM,IAAI,sBAAsB,CAC9B,oCAAoC,YAAY,sBAAsB,OAAO,gBAAgB,OAAO,OAAO,cAAc,CAAC,CAAC,CAAC,0JAA0J,CAAC,CAAC,CAAC,EAAE,EAAE,EAC7R,IAAI,EACJ,YAAY,EACZ,OAAO,EACP,oBAAoB,CACrB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-call timeout for readiness-poll Kubernetes API requests.
|
|
3
|
+
*
|
|
4
|
+
* Readiness polls loop as `while (Date.now() - startTime < timeout) { await k8sApi.read(...) }`. The
|
|
5
|
+
* deadline is only re-checked between iterations, so it relies on each `await` eventually settling. When
|
|
6
|
+
* the kubeconfig's exec credential WEDGES — e.g. `aws eks get-token` hangs, or the AWS/SSO session
|
|
7
|
+
* expired mid-deploy so the auth plugin never returns — the awaited request never resolves OR rejects,
|
|
8
|
+
* the loop never re-evaluates its deadline, and the whole deploy hangs SILENTLY (observed multi-hour).
|
|
9
|
+
*
|
|
10
|
+
* Bounding each call converts that wedge into a rejection, so the poll's existing error handling runs and
|
|
11
|
+
* the configured `timeout` is honored: a failed/expired credential is not a transient "not ready yet" —
|
|
12
|
+
* it counts against the deadline (or fails fast) instead of hanging forever.
|
|
13
|
+
*
|
|
14
|
+
* SCOPE / LIMITATION: this bounds the caller's `await` (so the poll and the deploy terminate). It does
|
|
15
|
+
* NOT cancel the in-flight request or kill a wedged exec-auth subprocess: `@kubernetes/client-node`'s
|
|
16
|
+
* `KubernetesObjectApi.read` does not thread an `AbortSignal` to its fetch, and its `ExecAuth` spawns the
|
|
17
|
+
* credential process (`child_process.spawn`) with no cancellation hook. A wedged subprocess can therefore
|
|
18
|
+
* keep the Node process alive until it exits or the process is terminated. The durable cure for the
|
|
19
|
+
* exec-auth failure mode is to avoid per-request exec auth during polling (e.g. a pre-minted bearer token
|
|
20
|
+
* in the kubeconfig), so no credential subprocess is spawned in the first place.
|
|
21
|
+
*/
|
|
22
|
+
/** Thrown when a readiness-poll API call exceeds its per-call budget (distinguishable so callers can fail fast vs. retry). */
|
|
23
|
+
export declare class PollTimeoutError extends Error {
|
|
24
|
+
readonly timeoutMs: number;
|
|
25
|
+
constructor(label: string, timeoutMs: number);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Run `op`, rejecting with {@link PollTimeoutError} if it hasn't settled within `timeoutMs`.
|
|
29
|
+
*
|
|
30
|
+
* PRECONDITION: `timeoutMs` must be positive — callers pass a per-call budget derived from the poll's
|
|
31
|
+
* REMAINING deadline (see {@link perCallTimeout}) and must handle an exhausted (≤0) budget themselves by
|
|
32
|
+
* exiting their loop and throwing their own overall-timeout error. A `PollTimeoutError` therefore always
|
|
33
|
+
* means "an operation that actually started did not return in time" (a wedged call), never "the overall
|
|
34
|
+
* poll deadline had already elapsed" — the two are distinct and must not be conflated.
|
|
35
|
+
*/
|
|
36
|
+
export declare function callWithTimeout<T>(op: () => Promise<T>, timeoutMs: number, label: string): Promise<T>;
|
|
37
|
+
/**
|
|
38
|
+
* The per-call budget: the configured cap (HTTP read timeout), capped STRICTLY by the poll's remaining
|
|
39
|
+
* deadline so one call can never overshoot the overall `timeout`. Returns ≤ 0 when the deadline is
|
|
40
|
+
* already spent; callers MUST treat a ≤0 budget as "deadline reached" — exit the poll loop and throw the
|
|
41
|
+
* poll's own overall-timeout error — rather than starting a (doomed) call. Never hand a ≤0 value to
|
|
42
|
+
* {@link callWithTimeout}.
|
|
43
|
+
*/
|
|
44
|
+
export declare function perCallTimeout(remainingMs: number, capMs: number): number;
|
|
45
|
+
//# sourceMappingURL=poll-timeout.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"poll-timeout.d.ts","sourceRoot":"","sources":["../../../src/core/deployment/poll-timeout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,8HAA8H;AAC9H,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBACf,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;CAS7C;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAY3G;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEzE"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-call timeout for readiness-poll Kubernetes API requests.
|
|
3
|
+
*
|
|
4
|
+
* Readiness polls loop as `while (Date.now() - startTime < timeout) { await k8sApi.read(...) }`. The
|
|
5
|
+
* deadline is only re-checked between iterations, so it relies on each `await` eventually settling. When
|
|
6
|
+
* the kubeconfig's exec credential WEDGES — e.g. `aws eks get-token` hangs, or the AWS/SSO session
|
|
7
|
+
* expired mid-deploy so the auth plugin never returns — the awaited request never resolves OR rejects,
|
|
8
|
+
* the loop never re-evaluates its deadline, and the whole deploy hangs SILENTLY (observed multi-hour).
|
|
9
|
+
*
|
|
10
|
+
* Bounding each call converts that wedge into a rejection, so the poll's existing error handling runs and
|
|
11
|
+
* the configured `timeout` is honored: a failed/expired credential is not a transient "not ready yet" —
|
|
12
|
+
* it counts against the deadline (or fails fast) instead of hanging forever.
|
|
13
|
+
*
|
|
14
|
+
* SCOPE / LIMITATION: this bounds the caller's `await` (so the poll and the deploy terminate). It does
|
|
15
|
+
* NOT cancel the in-flight request or kill a wedged exec-auth subprocess: `@kubernetes/client-node`'s
|
|
16
|
+
* `KubernetesObjectApi.read` does not thread an `AbortSignal` to its fetch, and its `ExecAuth` spawns the
|
|
17
|
+
* credential process (`child_process.spawn`) with no cancellation hook. A wedged subprocess can therefore
|
|
18
|
+
* keep the Node process alive until it exits or the process is terminated. The durable cure for the
|
|
19
|
+
* exec-auth failure mode is to avoid per-request exec auth during polling (e.g. a pre-minted bearer token
|
|
20
|
+
* in the kubeconfig), so no credential subprocess is spawned in the first place.
|
|
21
|
+
*/
|
|
22
|
+
/** Thrown when a readiness-poll API call exceeds its per-call budget (distinguishable so callers can fail fast vs. retry). */
|
|
23
|
+
export class PollTimeoutError extends Error {
|
|
24
|
+
timeoutMs;
|
|
25
|
+
constructor(label, timeoutMs) {
|
|
26
|
+
super(`${label} exceeded its ${timeoutMs}ms request timeout — the Kubernetes API call did not return. ` +
|
|
27
|
+
`The usual cause is a wedged or expired kubeconfig exec credential (e.g. an AWS SSO/EKS token ` +
|
|
28
|
+
`that expired mid-deploy). Re-run with fresh credentials.`);
|
|
29
|
+
this.name = 'PollTimeoutError';
|
|
30
|
+
this.timeoutMs = timeoutMs;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Run `op`, rejecting with {@link PollTimeoutError} if it hasn't settled within `timeoutMs`.
|
|
35
|
+
*
|
|
36
|
+
* PRECONDITION: `timeoutMs` must be positive — callers pass a per-call budget derived from the poll's
|
|
37
|
+
* REMAINING deadline (see {@link perCallTimeout}) and must handle an exhausted (≤0) budget themselves by
|
|
38
|
+
* exiting their loop and throwing their own overall-timeout error. A `PollTimeoutError` therefore always
|
|
39
|
+
* means "an operation that actually started did not return in time" (a wedged call), never "the overall
|
|
40
|
+
* poll deadline had already elapsed" — the two are distinct and must not be conflated.
|
|
41
|
+
*/
|
|
42
|
+
export async function callWithTimeout(op, timeoutMs, label) {
|
|
43
|
+
let timer;
|
|
44
|
+
try {
|
|
45
|
+
return await Promise.race([
|
|
46
|
+
op(),
|
|
47
|
+
new Promise((_, reject) => {
|
|
48
|
+
timer = setTimeout(() => reject(new PollTimeoutError(label, timeoutMs)), timeoutMs);
|
|
49
|
+
}),
|
|
50
|
+
]);
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
if (timer)
|
|
54
|
+
clearTimeout(timer);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The per-call budget: the configured cap (HTTP read timeout), capped STRICTLY by the poll's remaining
|
|
59
|
+
* deadline so one call can never overshoot the overall `timeout`. Returns ≤ 0 when the deadline is
|
|
60
|
+
* already spent; callers MUST treat a ≤0 budget as "deadline reached" — exit the poll loop and throw the
|
|
61
|
+
* poll's own overall-timeout error — rather than starting a (doomed) call. Never hand a ≤0 value to
|
|
62
|
+
* {@link callWithTimeout}.
|
|
63
|
+
*/
|
|
64
|
+
export function perCallTimeout(remainingMs, capMs) {
|
|
65
|
+
return Math.min(capMs, remainingMs);
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=poll-timeout.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"poll-timeout.js","sourceRoot":"","sources":["../../../src/core/deployment/poll-timeout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,8HAA8H;AAC9H,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,SAAS,CAAS;IAC3B,YAAY,KAAa,EAAE,SAAiB;QAC1C,KAAK,CACH,GAAG,KAAK,iBAAiB,SAAS,+DAA+D;YAC/F,+FAA+F;YAC/F,0DAA0D,CAC7D,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,EAAoB,EAAE,SAAiB,EAAE,KAAa;IAC7F,IAAI,KAAgD,CAAC;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;YACxB,EAAE,EAAE;YACJ,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACtF,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,WAAmB,EAAE,KAAa;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readiness.d.ts","sourceRoot":"","sources":["../../../src/core/deployment/readiness.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,KAAK,GAAG,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"readiness.d.ts","sourceRoot":"","sources":["../../../src/core/deployment/readiness.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,KAAK,GAAG,MAAM,yBAAyB,CAAC;AAUpD,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAmB,MAAM,wBAAwB,CAAC;AAClG,OAAO,KAAK,EACV,gBAAgB,EAChB,kBAAkB,EAEnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,qBAAa,wBAAwB;IAIvB,OAAO,CAAC,MAAM;IAH1B,OAAO,CAAC,eAAe,CAAC,CAAuC;IAC/D,OAAO,CAAC,WAAW,CAAC,CAAc;gBAEd,MAAM,EAAE,GAAG,CAAC,mBAAmB;IAEnD;;OAEG;IACH,kBAAkB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAIxE;;OAEG;IACH,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IAI9C;;OAEG;IACG,oBAAoB,CACxB,gBAAgB,EAAE,gBAAgB,EAClC,OAAO,EAAE,iBAAiB,EAC1B,SAAS,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,GAC1C,OAAO,CAAC,IAAI,CAAC;IAUhB;;OAEG;YACW,+BAA+B;IA6J7C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAIjC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO;IAYtD;;OAEG;IACH,OAAO,CAAC,sBAAsB;CAoB/B"}
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Handles checking if Kubernetes resources are ready after deployment
|
|
5
5
|
*/
|
|
6
|
-
import { DEFAULT_DEPLOYMENT_TIMEOUT, DEFAULT_FAST_POLL_INTERVAL, DEFAULT_POLL_INTERVAL, DEFAULT_READINESS_MAX_BACKOFF, } from '../config/defaults.js';
|
|
6
|
+
import { DEFAULT_DEPLOYMENT_TIMEOUT, DEFAULT_FAST_POLL_INTERVAL, DEFAULT_HTTP_READ_TIMEOUT, DEFAULT_POLL_INTERVAL, DEFAULT_READINESS_MAX_BACKOFF, } from '../config/defaults.js';
|
|
7
7
|
import { ensureError } from '../errors.js';
|
|
8
|
+
import { callWithTimeout, perCallTimeout } from './poll-timeout.js';
|
|
8
9
|
import { ResourceReadinessTimeoutError } from './errors.js';
|
|
9
10
|
export class ResourceReadinessChecker {
|
|
10
11
|
k8sApi;
|
|
@@ -38,6 +39,9 @@ export class ResourceReadinessChecker {
|
|
|
38
39
|
async waitForResourceReadyWithPolling(deployedResource, _options, readinessConfig, emitEvent) {
|
|
39
40
|
const startTime = Date.now();
|
|
40
41
|
let attempt = 0;
|
|
42
|
+
// The most recent successfully-read resource, reused for the on-timeout debug log so we never issue a
|
|
43
|
+
// fresh (post-deadline) read that could hang or delay the timeout error.
|
|
44
|
+
let lastObserved;
|
|
41
45
|
emitEvent({
|
|
42
46
|
type: 'progress',
|
|
43
47
|
resourceId: deployedResource.id,
|
|
@@ -47,15 +51,23 @@ export class ResourceReadinessChecker {
|
|
|
47
51
|
while (Date.now() - startTime < readinessConfig.timeout) {
|
|
48
52
|
attempt++;
|
|
49
53
|
try {
|
|
50
|
-
// In the new API, methods return objects directly (no .body wrapper)
|
|
51
|
-
|
|
54
|
+
// In the new API, methods return objects directly (no .body wrapper).
|
|
55
|
+
// Bound the read so a wedged/expired kubeconfig exec credential rejects (→ handled below, retried
|
|
56
|
+
// within the deadline) instead of hanging the poll forever — see poll-timeout.ts.
|
|
57
|
+
const readTimeout = perCallTimeout(readinessConfig.timeout - (Date.now() - startTime), DEFAULT_HTTP_READ_TIMEOUT);
|
|
58
|
+
// Deadline spent: stop and fall through to ResourceReadinessTimeoutError (the poll's own overall
|
|
59
|
+
// timeout) rather than starting a doomed read.
|
|
60
|
+
if (readTimeout <= 0)
|
|
61
|
+
break;
|
|
62
|
+
const currentResource = await callWithTimeout(() => this.k8sApi.read({
|
|
52
63
|
apiVersion: deployedResource.manifest.apiVersion,
|
|
53
64
|
kind: deployedResource.kind,
|
|
54
65
|
metadata: {
|
|
55
66
|
name: deployedResource.name,
|
|
56
67
|
namespace: deployedResource.namespace,
|
|
57
68
|
},
|
|
58
|
-
});
|
|
69
|
+
}), readTimeout, `read ${deployedResource.kind}/${deployedResource.name}`);
|
|
70
|
+
lastObserved = currentResource;
|
|
59
71
|
const isReady = this.isResourceReady(currentResource);
|
|
60
72
|
// Debug logging for status polling
|
|
61
73
|
if (this.debugLogger) {
|
|
@@ -112,27 +124,24 @@ export class ResourceReadinessChecker {
|
|
|
112
124
|
timestamp: new Date(),
|
|
113
125
|
});
|
|
114
126
|
}
|
|
115
|
-
// Wait before retry
|
|
116
|
-
|
|
127
|
+
// Wait before retry — but never past the deadline. If a bounded read consumed the remaining
|
|
128
|
+
// budget, sleeping the full errorRetryDelay would overshoot the configured timeout; cap it to
|
|
129
|
+
// what's left (0 → the while re-check exits immediately).
|
|
130
|
+
const retryDelay = Math.max(0, Math.min(readinessConfig.errorRetryDelay, readinessConfig.timeout - (Date.now() - startTime)));
|
|
131
|
+
if (retryDelay > 0) {
|
|
132
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
133
|
+
}
|
|
117
134
|
}
|
|
118
135
|
}
|
|
119
|
-
// Debug logging for timeout with final status
|
|
136
|
+
// Debug logging for timeout with final status. Use the LAST OBSERVED resource rather than issuing a
|
|
137
|
+
// fresh read: the deadline is already spent, so a new read would either hang (wedged credential — the
|
|
138
|
+
// very failure mode this fixes) or delay ResourceReadinessTimeoutError past the configured timeout.
|
|
120
139
|
if (this.debugLogger) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
const finalResource = await this.k8sApi.read({
|
|
124
|
-
apiVersion: deployedResource.manifest.apiVersion,
|
|
125
|
-
kind: deployedResource.kind,
|
|
126
|
-
metadata: {
|
|
127
|
-
name: deployedResource.name,
|
|
128
|
-
namespace: deployedResource.namespace,
|
|
129
|
-
},
|
|
130
|
-
});
|
|
131
|
-
this.debugLogger.logTimeout(deployedResource, finalResource.status || finalResource, Date.now() - startTime, attempt);
|
|
140
|
+
if (lastObserved !== undefined) {
|
|
141
|
+
this.debugLogger.logTimeout(deployedResource, lastObserved.status || lastObserved, Date.now() - startTime, attempt);
|
|
132
142
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
this.debugLogger.logTimeout(deployedResource, { error: 'Could not retrieve final status' }, Date.now() - startTime, attempt);
|
|
143
|
+
else {
|
|
144
|
+
this.debugLogger.logTimeout(deployedResource, { error: 'No status observed before timeout' }, Date.now() - startTime, attempt);
|
|
136
145
|
}
|
|
137
146
|
}
|
|
138
147
|
throw new ResourceReadinessTimeoutError(deployedResource, readinessConfig.timeout);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readiness.js","sourceRoot":"","sources":["../../../src/core/deployment/readiness.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,qBAAqB,EACrB,6BAA6B,GAC9B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"readiness.js","sourceRoot":"","sources":["../../../src/core/deployment/readiness.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,6BAA6B,GAC9B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAQpE,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAE5D,MAAM,OAAO,wBAAwB;IAIf;IAHZ,eAAe,CAAwC;IACvD,WAAW,CAAe;IAElC,YAAoB,MAA+B;QAA/B,WAAM,GAAN,MAAM,CAAyB;IAAG,CAAC;IAEvD;;OAEG;IACH,kBAAkB,CAAC,QAA8C;QAC/D,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,WAAwB;QACrC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CACxB,gBAAkC,EAClC,OAA0B,EAC1B,SAA2C;QAE3C,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,+BAA+B,CACxC,gBAAgB,EAChB,OAAO,EACP,eAAe,EACf,SAAS,CACV,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,+BAA+B,CAC3C,gBAAkC,EAClC,QAA2B,EAC3B,eAAgC,EAChC,SAA2C;QAE3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,sGAAsG;QACtG,yEAAyE;QACzE,IAAI,YAA8C,CAAC;QAEnD,SAAS,CAAC;YACR,IAAI,EAAE,UAAU;YAChB,UAAU,EAAE,gBAAgB,CAAC,EAAE;YAC/B,OAAO,EAAE,eAAe,gBAAgB,CAAC,IAAI,IAAI,gBAAgB,CAAC,IAAI,cAAc;YACpF,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;YACxD,OAAO,EAAE,CAAC;YAEV,IAAI,CAAC;gBACH,sEAAsE;gBACtE,kGAAkG;gBAClG,kFAAkF;gBAClF,MAAM,WAAW,GAAG,cAAc,CAChC,eAAe,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,EAClD,yBAAyB,CAC1B,CAAC;gBACF,iGAAiG;gBACjG,+CAA+C;gBAC/C,IAAI,WAAW,IAAI,CAAC;oBAAE,MAAM;gBAC5B,MAAM,eAAe,GAAG,MAAM,eAAe,CAC3C,GAAG,EAAE,CACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,gBAAgB,CAAC,QAAQ,CAAC,UAAU;oBAChD,IAAI,EAAE,gBAAgB,CAAC,IAAI;oBAC3B,QAAQ,EAAE;wBACR,IAAI,EAAE,gBAAgB,CAAC,IAAI;wBAC3B,SAAS,EAAE,gBAAgB,CAAC,SAAS;qBACtC;iBACF,CAAC,EACJ,WAAW,EACX,QAAQ,gBAAgB,CAAC,IAAI,IAAI,gBAAgB,CAAC,IAAI,EAAE,CACzD,CAAC;gBACF,YAAY,GAAG,eAAe,CAAC;gBAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;gBAEtD,mCAAmC;gBACnC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAChC,gBAAgB,EACf,eAAwC,CAAC,MAAM,IAAI,eAAe,EACnE,OAAO,EACP;wBACE,OAAO;wBACP,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACnC,SAAS,EAAE,KAAK;wBAChB,gBAAgB,EAAE,SAAS;qBAC5B,CACF,CAAC;gBACJ,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAExC,6DAA6D;oBAC7D,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;wBACzB,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;oBACzC,CAAC;oBAED,SAAS,CAAC;wBACR,IAAI,EAAE,UAAU;wBAChB,UAAU,EAAE,gBAAgB,CAAC,EAAE;wBAC/B,OAAO,EAAE,GAAG,gBAAgB,CAAC,IAAI,IAAI,gBAAgB,CAAC,IAAI,mBAAmB,QAAQ,IAAI;wBACzF,SAAS,EAAE,IAAI,IAAI,EAAE;qBACtB,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,uBAAuB;gBACvB,IAAI,OAAO,GAAG,eAAe,CAAC,gBAAgB,KAAK,CAAC,EAAE,CAAC;oBACrD,SAAS,CAAC;wBACR,IAAI,EAAE,UAAU;wBAChB,UAAU,EAAE,gBAAgB,CAAC,EAAE;wBAC/B,OAAO,EAAE,qBAAqB,gBAAgB,CAAC,IAAI,IAAI,gBAAgB,CAAC,IAAI,aAAa,OAAO,GAAG;wBACnG,SAAS,EAAE,IAAI,IAAI,EAAE;qBACtB,CAAC,CAAC;gBACL,CAAC;gBAED,kDAAkD;gBAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,eAAe,CAAC,YAAY,GAAG,eAAe,CAAC,iBAAiB,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,EACjF,eAAe,CAAC,QAAQ,CACzB,CAAC;gBAEF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,+BAA+B;gBAC/B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,gBAAgB,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE;wBACjE,OAAO;wBACP,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACnC,SAAS,EAAE,KAAK;wBAChB,gBAAgB,EAAE,SAAS;qBAC5B,CAAC,CAAC;gBACL,CAAC;gBAED,iCAAiC;gBACjC,IAAI,OAAO,GAAG,eAAe,CAAC,gBAAgB,KAAK,CAAC,EAAE,CAAC;oBACrD,SAAS,CAAC;wBACR,IAAI,EAAE,UAAU;wBAChB,UAAU,EAAE,gBAAgB,CAAC,EAAE;wBAC/B,OAAO,EAAE,gCAAgC,gBAAgB,CAAC,IAAI,IAAI,gBAAgB,CAAC,IAAI,KAAK,WAAW,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;wBACxH,SAAS,EAAE,IAAI,IAAI,EAAE;qBACtB,CAAC,CAAC;gBACL,CAAC;gBAED,4FAA4F;gBAC5F,8FAA8F;gBAC9F,0DAA0D;gBAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,eAAe,EAAE,eAAe,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAC9F,CAAC;gBACF,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACnB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;gBAClE,CAAC;YACH,CAAC;QACH,CAAC;QAED,oGAAoG;QACpG,sGAAsG;QACtG,oGAAoG;QACpG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC/B,IAAI,CAAC,WAAW,CAAC,UAAU,CACzB,gBAAgB,EACf,YAAqC,CAAC,MAAM,IAAI,YAAY,EAC7D,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EACtB,OAAO,CACR,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,WAAW,CAAC,UAAU,CACzB,gBAAgB,EAChB,EAAE,KAAK,EAAE,mCAAmC,EAAE,EAC9C,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EACtB,OAAO,CACR,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,IAAI,6BAA6B,CAAC,gBAAgB,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,OAA0B;QACnD,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,0BAA0B;YACtD,YAAY,EAAE,0BAA0B;YACxC,QAAQ,EAAE,6BAA6B;YACvC,iBAAiB,EAAE,GAAG;YACtB,eAAe,EAAE,qBAAqB;YACtC,gBAAgB,EAAE,CAAC,EAAE,iCAAiC;SACvD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,yBAAyB,CAAC,GAAyB,EAAE,KAAa;QACxE,OAAQ,GAA+B,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACK,iBAAiB,CACvB,QAA4B;QAE5B,IAAI,QAAQ,IAAI,QAAQ,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC/C,0EAA0E;YAC1E,OAAO,QAAQ,CAAC,MAAuB,CAAC;QAC1C,CAAC;QACD,qDAAqD;QACrD,OAAO,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAkB,CAAC;IAC7E,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,QAA4B;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEhD,+DAA+D;QAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iEAAiE;QACjE,OAAO,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,QAA4B;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAA2B,QAAQ,CAAC,CAAC;QAE1E,wCAAwC;QACxC,IAAI,MAAM,EAAE,UAAU,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;YACzE,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,cAAc,CAAC,MAAM,KAAK,MAAM,CAAC;YAC1C,CAAC;YAED,gCAAgC;YAChC,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YACjF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,OAAO,kBAAkB,CAAC,MAAM,KAAK,MAAM,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|