pepr 0.34.1 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/init/templates.d.ts +1 -0
- package/dist/cli/init/templates.d.ts.map +1 -1
- package/dist/cli.js +48 -23
- package/dist/controller.js +1 -1
- package/dist/lib/assets/helm.d.ts.map +1 -1
- package/dist/lib/assets/pods.d.ts.map +1 -1
- package/dist/lib/assets/yaml.d.ts.map +1 -1
- package/dist/lib/capability.d.ts.map +1 -1
- package/dist/lib/filter.d.ts.map +1 -1
- package/dist/lib/helpers.d.ts.map +1 -1
- package/dist/lib/logger.d.ts +1 -1
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/queue.d.ts +19 -3
- package/dist/lib/queue.d.ts.map +1 -1
- package/dist/lib/types.d.ts +3 -0
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/watch-processor.d.ts +10 -1
- package/dist/lib/watch-processor.d.ts.map +1 -1
- package/dist/lib.js +104 -32
- package/dist/lib.js.map +3 -3
- package/dist/sdk/sdk.d.ts +5 -3
- package/dist/sdk/sdk.d.ts.map +1 -1
- package/package.json +10 -10
- package/src/cli/init/templates.ts +1 -0
- package/src/lib/assets/helm.ts +4 -16
- package/src/lib/assets/pods.ts +4 -0
- package/src/lib/assets/yaml.ts +32 -0
- package/src/lib/capability.ts +9 -1
- package/src/lib/filter.test.ts +85 -1
- package/src/lib/filter.ts +9 -0
- package/src/lib/helpers.test.ts +34 -0
- package/src/lib/helpers.ts +5 -0
- package/src/lib/queue.test.ts +138 -44
- package/src/lib/queue.ts +48 -13
- package/src/lib/types.ts +3 -0
- package/src/lib/watch-processor.test.ts +101 -5
- package/src/lib/watch-processor.ts +49 -16
- package/src/sdk/sdk.test.ts +46 -13
- package/src/sdk/sdk.ts +15 -6
- package/src/templates/capabilities/hello-pepr.ts +9 -0
package/src/lib/queue.test.ts
CHANGED
|
@@ -1,58 +1,152 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { KubernetesObject } from "@kubernetes/client-node";
|
|
1
|
+
import { afterEach, describe, expect, jest, it } from "@jest/globals";
|
|
3
2
|
import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types";
|
|
4
3
|
import { Queue } from "./queue";
|
|
5
4
|
|
|
5
|
+
import Log from "./logger";
|
|
6
|
+
jest.mock("./logger");
|
|
7
|
+
|
|
6
8
|
describe("Queue", () => {
|
|
7
|
-
|
|
9
|
+
afterEach(() => {
|
|
10
|
+
jest.resetAllMocks();
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("is uniquely identifiable", () => {
|
|
14
|
+
const name = "kind/namespace";
|
|
15
|
+
const queue = new Queue(name);
|
|
16
|
+
|
|
17
|
+
const label = queue.label();
|
|
18
|
+
|
|
19
|
+
expect(label).toEqual(
|
|
20
|
+
expect.objectContaining({
|
|
21
|
+
// given name of queue
|
|
22
|
+
name,
|
|
23
|
+
|
|
24
|
+
// unique, generated value (to disambiguate similarly-named queues)
|
|
25
|
+
// <epoch timestamp (ms)>-<4 char hex>
|
|
26
|
+
uid: expect.stringMatching(/[0-9]{13}-[0-9a-f]{4}/),
|
|
27
|
+
}),
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it("exposes runtime stats", async () => {
|
|
32
|
+
const name = "kind/namespace";
|
|
33
|
+
const queue = new Queue(name);
|
|
34
|
+
|
|
35
|
+
expect(queue.stats()).toEqual(
|
|
36
|
+
expect.objectContaining({
|
|
37
|
+
queue: queue.label(),
|
|
38
|
+
stats: { length: 0 },
|
|
39
|
+
}),
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
const kubeObj = { metadata: { name: "test-nm", namespace: "test-ns" } };
|
|
43
|
+
const watchCb = () =>
|
|
44
|
+
new Promise<void>(res => {
|
|
45
|
+
setTimeout(res, 100);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
await Promise.all([
|
|
49
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchCb),
|
|
50
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchCb),
|
|
51
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchCb),
|
|
52
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchCb),
|
|
53
|
+
]);
|
|
54
|
+
|
|
55
|
+
const logDebug = Log.debug as jest.Mock;
|
|
56
|
+
const stats = logDebug.mock.calls
|
|
57
|
+
.flat()
|
|
58
|
+
.map(m => JSON.stringify(m))
|
|
59
|
+
.filter(m => m.includes('"stats":'));
|
|
8
60
|
|
|
9
|
-
|
|
10
|
-
|
|
61
|
+
[
|
|
62
|
+
'"length":1', // 1st entry runs near-immediately, so queue won't fill
|
|
63
|
+
'"length":1', // afterward, queue fills & unfills as callbacks process
|
|
64
|
+
'"length":2',
|
|
65
|
+
'"length":3',
|
|
66
|
+
'"length":3',
|
|
67
|
+
'"length":2',
|
|
68
|
+
'"length":1',
|
|
69
|
+
'"length":0',
|
|
70
|
+
].map((exp, idx) => {
|
|
71
|
+
expect(stats[idx]).toEqual(expect.stringContaining(exp));
|
|
72
|
+
});
|
|
11
73
|
});
|
|
12
74
|
|
|
13
|
-
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const
|
|
75
|
+
it("resolves when an enqueued event dequeues without error", async () => {
|
|
76
|
+
const name = "kind/namespace";
|
|
77
|
+
const queue = new Queue(name);
|
|
78
|
+
|
|
79
|
+
const kubeObj = { metadata: { name: "test-nm", namespace: "test-ns" } };
|
|
80
|
+
const watchCb = () =>
|
|
81
|
+
new Promise<void>(res => {
|
|
82
|
+
setTimeout(res, 10);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
const promise = queue.enqueue(kubeObj, WatchPhase.Added, watchCb);
|
|
18
86
|
expect(promise).toBeInstanceOf(Promise);
|
|
19
|
-
|
|
87
|
+
|
|
88
|
+
await expect(promise).resolves.not.toThrow();
|
|
20
89
|
});
|
|
21
90
|
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
91
|
+
it("rejects when an enqueued event dequeues with error", async () => {
|
|
92
|
+
const name = "kind/namespace";
|
|
93
|
+
const queue = new Queue(name);
|
|
94
|
+
|
|
95
|
+
const kubeObj = { metadata: { name: "test-nm", namespace: "test-ns" } };
|
|
96
|
+
const watchCb = () =>
|
|
97
|
+
new Promise<void>((_, reject) => {
|
|
98
|
+
setTimeout(() => {
|
|
99
|
+
reject("oof");
|
|
100
|
+
}, 10);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
const promise = queue.enqueue(kubeObj, WatchPhase.Added, watchCb);
|
|
104
|
+
expect(promise).toBeInstanceOf(Promise);
|
|
105
|
+
|
|
106
|
+
await expect(promise).rejects.toBe("oof");
|
|
37
107
|
});
|
|
38
108
|
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
109
|
+
it("processes events in FIFO order", async () => {
|
|
110
|
+
const name = "kind/namespace";
|
|
111
|
+
const queue = new Queue(name);
|
|
112
|
+
|
|
113
|
+
const kubeObj = { metadata: { name: "test-nm", namespace: "test-ns" } };
|
|
114
|
+
const watchA = () =>
|
|
115
|
+
new Promise<void>(resolve => {
|
|
116
|
+
setTimeout(() => {
|
|
117
|
+
Log.info("watchA");
|
|
118
|
+
resolve();
|
|
119
|
+
}, 15);
|
|
120
|
+
});
|
|
121
|
+
const watchB = () =>
|
|
122
|
+
new Promise<void>(resolve => {
|
|
123
|
+
setTimeout(() => {
|
|
124
|
+
Log.info("watchB");
|
|
125
|
+
resolve();
|
|
126
|
+
}, 10);
|
|
127
|
+
});
|
|
128
|
+
const watchC = () =>
|
|
129
|
+
new Promise<void>(resolve => {
|
|
130
|
+
setTimeout(() => {
|
|
131
|
+
Log.info("watchC");
|
|
132
|
+
resolve();
|
|
133
|
+
}, 5);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
await Promise.all([
|
|
137
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchA),
|
|
138
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchB),
|
|
139
|
+
queue.enqueue(kubeObj, WatchPhase.Added, watchC),
|
|
140
|
+
]);
|
|
141
|
+
|
|
142
|
+
const logInfo = Log.info as jest.Mock;
|
|
143
|
+
const calls = logInfo.mock.calls
|
|
144
|
+
.flat()
|
|
145
|
+
.map(m => JSON.stringify(m))
|
|
146
|
+
.filter(m => /"watch[ABC]"/.test(m));
|
|
147
|
+
|
|
148
|
+
['"watchA"', '"watchB"', '"watchC"'].map((exp, idx) => {
|
|
149
|
+
expect(calls[idx]).toEqual(expect.stringContaining(exp));
|
|
150
|
+
});
|
|
57
151
|
});
|
|
58
152
|
});
|
package/src/lib/queue.ts
CHANGED
|
@@ -2,11 +2,15 @@
|
|
|
2
2
|
// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
|
|
3
3
|
import { KubernetesObject } from "@kubernetes/client-node";
|
|
4
4
|
import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types";
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
5
6
|
import Log from "./logger";
|
|
6
7
|
|
|
8
|
+
type WatchCallback = (obj: KubernetesObject, phase: WatchPhase) => Promise<void>;
|
|
9
|
+
|
|
7
10
|
type QueueItem<K extends KubernetesObject> = {
|
|
8
11
|
item: K;
|
|
9
|
-
|
|
12
|
+
phase: WatchPhase;
|
|
13
|
+
callback: WatchCallback;
|
|
10
14
|
resolve: (value: void | PromiseLike<void>) => void;
|
|
11
15
|
reject: (reason?: string) => void;
|
|
12
16
|
};
|
|
@@ -15,16 +19,27 @@ type QueueItem<K extends KubernetesObject> = {
|
|
|
15
19
|
* Queue is a FIFO queue for reconciling
|
|
16
20
|
*/
|
|
17
21
|
export class Queue<K extends KubernetesObject> {
|
|
22
|
+
#name: string;
|
|
23
|
+
#uid: string;
|
|
18
24
|
#queue: QueueItem<K>[] = [];
|
|
19
25
|
#pendingPromise = false;
|
|
20
|
-
#reconcile?: (obj: KubernetesObject, type: WatchPhase) => Promise<void>;
|
|
21
26
|
|
|
22
|
-
constructor() {
|
|
23
|
-
this.#
|
|
27
|
+
constructor(name: string) {
|
|
28
|
+
this.#name = name;
|
|
29
|
+
this.#uid = `${Date.now()}-${randomBytes(2).toString("hex")}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
label() {
|
|
33
|
+
return { name: this.#name, uid: this.#uid };
|
|
24
34
|
}
|
|
25
35
|
|
|
26
|
-
|
|
27
|
-
|
|
36
|
+
stats() {
|
|
37
|
+
return {
|
|
38
|
+
queue: this.label(),
|
|
39
|
+
stats: {
|
|
40
|
+
length: this.#queue.length,
|
|
41
|
+
},
|
|
42
|
+
};
|
|
28
43
|
}
|
|
29
44
|
|
|
30
45
|
/**
|
|
@@ -32,12 +47,23 @@ export class Queue<K extends KubernetesObject> {
|
|
|
32
47
|
* reconciled.
|
|
33
48
|
*
|
|
34
49
|
* @param item The object to reconcile
|
|
50
|
+
* @param type The watch phase requested for reconcile
|
|
51
|
+
* @param reconcile The callback to enqueue for reconcile
|
|
35
52
|
* @returns A promise that resolves when the object is reconciled
|
|
36
53
|
*/
|
|
37
|
-
enqueue(item: K,
|
|
38
|
-
|
|
54
|
+
enqueue(item: K, phase: WatchPhase, reconcile: WatchCallback) {
|
|
55
|
+
const note = {
|
|
56
|
+
queue: this.label(),
|
|
57
|
+
item: {
|
|
58
|
+
name: item.metadata?.name,
|
|
59
|
+
namespace: item.metadata?.namespace,
|
|
60
|
+
resourceVersion: item.metadata?.resourceVersion,
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
Log.debug(note, "Enqueueing");
|
|
39
64
|
return new Promise<void>((resolve, reject) => {
|
|
40
|
-
this.#queue.push({ item,
|
|
65
|
+
this.#queue.push({ item, phase, callback: reconcile, resolve, reject });
|
|
66
|
+
Log.debug(this.stats(), "Queue stats - push");
|
|
41
67
|
return this.#dequeue();
|
|
42
68
|
});
|
|
43
69
|
}
|
|
@@ -68,16 +94,25 @@ export class Queue<K extends KubernetesObject> {
|
|
|
68
94
|
this.#pendingPromise = true;
|
|
69
95
|
|
|
70
96
|
// Reconcile the element
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
97
|
+
const note = {
|
|
98
|
+
queue: this.label(),
|
|
99
|
+
item: {
|
|
100
|
+
name: element.item.metadata?.name,
|
|
101
|
+
namespace: element.item.metadata?.namespace,
|
|
102
|
+
resourceVersion: element.item.metadata?.resourceVersion,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
Log.debug(note, "Reconciling");
|
|
106
|
+
await element.callback(element.item, element.phase);
|
|
107
|
+
Log.debug(note, "Reconciled");
|
|
75
108
|
|
|
76
109
|
element.resolve();
|
|
77
110
|
} catch (e) {
|
|
78
111
|
Log.debug(`Error reconciling ${element.item.metadata!.name}`, { error: e });
|
|
79
112
|
element.reject(e);
|
|
80
113
|
} finally {
|
|
114
|
+
Log.debug(this.stats(), "Queue stats - shift");
|
|
115
|
+
|
|
81
116
|
// Reset the pending promise flag
|
|
82
117
|
Log.debug("Resetting pending promise and dequeuing");
|
|
83
118
|
this.#pendingPromise = false;
|
package/src/lib/types.ts
CHANGED
|
@@ -94,6 +94,7 @@ export type Binding = {
|
|
|
94
94
|
namespaces: string[];
|
|
95
95
|
labels: Record<string, string>;
|
|
96
96
|
annotations: Record<string, string>;
|
|
97
|
+
deletionTimestamp: boolean;
|
|
97
98
|
};
|
|
98
99
|
readonly mutateCallback?: MutateAction<GenericClass, InstanceType<GenericClass>>;
|
|
99
100
|
readonly validateCallback?: ValidateAction<GenericClass, InstanceType<GenericClass>>;
|
|
@@ -137,6 +138,8 @@ export type BindingFilter<T extends GenericClass> = CommonActionChain<T> & {
|
|
|
137
138
|
* @param value
|
|
138
139
|
*/
|
|
139
140
|
WithAnnotation: (key: string, value?: string) => BindingFilter<T>;
|
|
141
|
+
/** Only apply the action if the resource has a deletionTimestamp. */
|
|
142
|
+
WithDeletionTimestamp: () => BindingFilter<T>;
|
|
140
143
|
};
|
|
141
144
|
|
|
142
145
|
export type BindingWithName<T extends GenericClass> = BindingFilter<T> & {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
|
|
3
|
-
import { beforeEach, describe, expect, it, jest } from "@jest/globals";
|
|
3
|
+
import { afterAll, beforeEach, describe, expect, it, jest } from "@jest/globals";
|
|
4
4
|
import { GenericClass, K8s, KubernetesObject, kind } from "kubernetes-fluent-client";
|
|
5
5
|
import { K8sInit, WatchPhase } from "kubernetes-fluent-client/dist/fluent/types";
|
|
6
6
|
import { WatchCfg, WatchEvent, Watcher } from "kubernetes-fluent-client/dist/fluent/watch";
|
|
7
7
|
import { Capability } from "./capability";
|
|
8
|
-
import { setupWatch, logEvent } from "./watch-processor";
|
|
8
|
+
import { setupWatch, logEvent, queueKey, getOrCreateQueue } from "./watch-processor";
|
|
9
9
|
import Log from "./logger";
|
|
10
10
|
import { metricsCollector } from "./metrics";
|
|
11
11
|
|
|
@@ -40,6 +40,7 @@ describe("WatchProcessor", () => {
|
|
|
40
40
|
bindings: [
|
|
41
41
|
{
|
|
42
42
|
isWatch: true,
|
|
43
|
+
isQueue: false,
|
|
43
44
|
model: "someModel",
|
|
44
45
|
filters: {},
|
|
45
46
|
event: "Create",
|
|
@@ -88,13 +89,20 @@ describe("WatchProcessor", () => {
|
|
|
88
89
|
resyncFailureMax: 5,
|
|
89
90
|
resyncDelaySec: 5,
|
|
90
91
|
lastSeenLimitSeconds: 300,
|
|
91
|
-
relistIntervalSec:
|
|
92
|
+
relistIntervalSec: 600,
|
|
92
93
|
};
|
|
93
94
|
|
|
94
95
|
capabilities.push({
|
|
95
96
|
bindings: [
|
|
96
|
-
{
|
|
97
|
-
|
|
97
|
+
{
|
|
98
|
+
isWatch: true,
|
|
99
|
+
isQueue: true,
|
|
100
|
+
model: "someModel",
|
|
101
|
+
filters: { name: "bleh" },
|
|
102
|
+
event: "Create",
|
|
103
|
+
watchCallback: jest.fn(),
|
|
104
|
+
},
|
|
105
|
+
{ isWatch: false, isQueue: false, model: "someModel", filters: {}, event: "Create", watchCallback: jest.fn() },
|
|
98
106
|
],
|
|
99
107
|
} as unknown as Capability);
|
|
100
108
|
|
|
@@ -320,3 +328,91 @@ describe("logEvent function", () => {
|
|
|
320
328
|
expect(Log.debug).toHaveBeenCalledWith(`Watch event ${WatchEvent.DATA_ERROR} received. ${message}.`);
|
|
321
329
|
});
|
|
322
330
|
});
|
|
331
|
+
|
|
332
|
+
describe("queueKey", () => {
|
|
333
|
+
const withKindNsName = { kind: "Pod", metadata: { namespace: "my-ns", name: "my-name" } } as KubernetesObject;
|
|
334
|
+
const withKindNs = { kind: "Pod", metadata: { namespace: "my-ns" } } as KubernetesObject;
|
|
335
|
+
const withKindName = { kind: "Pod", metadata: { name: "my-name" } } as KubernetesObject;
|
|
336
|
+
const withNsName = { metadata: { namespace: "my-ns", name: "my-name" } } as KubernetesObject;
|
|
337
|
+
const withKind = { kind: "Pod" } as KubernetesObject;
|
|
338
|
+
const withNs = { metadata: { namespace: "my-ns" } } as KubernetesObject;
|
|
339
|
+
const withName = { metadata: { name: "my-name" } } as KubernetesObject;
|
|
340
|
+
const withNone = {} as KubernetesObject;
|
|
341
|
+
|
|
342
|
+
const original = process.env.PEPR_RECONCILE_STRATEGY;
|
|
343
|
+
|
|
344
|
+
it.each([
|
|
345
|
+
["kind", withKindNsName, "Pod"],
|
|
346
|
+
["kind", withKindNs, "Pod"],
|
|
347
|
+
["kind", withKindName, "Pod"],
|
|
348
|
+
["kind", withNsName, "UnknownKind"],
|
|
349
|
+
["kind", withKind, "Pod"],
|
|
350
|
+
["kind", withNs, "UnknownKind"],
|
|
351
|
+
["kind", withName, "UnknownKind"],
|
|
352
|
+
["kind", withNone, "UnknownKind"],
|
|
353
|
+
["kindNs", withKindNsName, "Pod/my-ns"],
|
|
354
|
+
["kindNs", withKindNs, "Pod/my-ns"],
|
|
355
|
+
["kindNs", withKindName, "Pod/cluster-scoped"],
|
|
356
|
+
["kindNs", withNsName, "UnknownKind/my-ns"],
|
|
357
|
+
["kindNs", withKind, "Pod/cluster-scoped"],
|
|
358
|
+
["kindNs", withNs, "UnknownKind/my-ns"],
|
|
359
|
+
["kindNs", withName, "UnknownKind/cluster-scoped"],
|
|
360
|
+
["kindNs", withNone, "UnknownKind/cluster-scoped"],
|
|
361
|
+
["kindNsName", withKindNsName, "Pod/my-ns/my-name"],
|
|
362
|
+
["kindNsName", withKindNs, "Pod/my-ns/Unnamed"],
|
|
363
|
+
["kindNsName", withKindName, "Pod/cluster-scoped/my-name"],
|
|
364
|
+
["kindNsName", withNsName, "UnknownKind/my-ns/my-name"],
|
|
365
|
+
["kindNsName", withKind, "Pod/cluster-scoped/Unnamed"],
|
|
366
|
+
["kindNsName", withNs, "UnknownKind/my-ns/Unnamed"],
|
|
367
|
+
["kindNsName", withName, "UnknownKind/cluster-scoped/my-name"],
|
|
368
|
+
["kindNsName", withNone, "UnknownKind/cluster-scoped/Unnamed"],
|
|
369
|
+
["global", withKindNsName, "global"],
|
|
370
|
+
["global", withKindNs, "global"],
|
|
371
|
+
["global", withKindName, "global"],
|
|
372
|
+
["global", withNsName, "global"],
|
|
373
|
+
["global", withKind, "global"],
|
|
374
|
+
["global", withNs, "global"],
|
|
375
|
+
["global", withName, "global"],
|
|
376
|
+
["global", withNone, "global"],
|
|
377
|
+
])("PEPR_RECONCILE_STRATEGY='%s' over '%j' becomes '%s'", (strat, obj, key) => {
|
|
378
|
+
process.env.PEPR_RECONCILE_STRATEGY = strat;
|
|
379
|
+
expect(queueKey(obj)).toBe(key);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
afterAll(() => {
|
|
383
|
+
process.env.PEPR_RECONCILE_STRATEGY = original;
|
|
384
|
+
});
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
describe("getOrCreateQueue", () => {
|
|
388
|
+
it("creates a Queue instance on first call", () => {
|
|
389
|
+
const obj: KubernetesObject = {
|
|
390
|
+
kind: "queue",
|
|
391
|
+
metadata: {
|
|
392
|
+
name: "nm",
|
|
393
|
+
namespace: "ns",
|
|
394
|
+
},
|
|
395
|
+
};
|
|
396
|
+
|
|
397
|
+
const firstQueue = getOrCreateQueue(obj);
|
|
398
|
+
expect(firstQueue.label()).toBeDefined();
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
it("returns same Queue instance on subsequent calls", () => {
|
|
402
|
+
const obj: KubernetesObject = {
|
|
403
|
+
kind: "queue",
|
|
404
|
+
metadata: {
|
|
405
|
+
name: "nm",
|
|
406
|
+
namespace: "ns",
|
|
407
|
+
},
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
const firstQueue = getOrCreateQueue(obj);
|
|
411
|
+
expect(firstQueue.label()).toBeDefined();
|
|
412
|
+
|
|
413
|
+
const secondQueue = getOrCreateQueue(obj);
|
|
414
|
+
expect(secondQueue.label()).toBeDefined();
|
|
415
|
+
|
|
416
|
+
expect(firstQueue).toBe(secondQueue);
|
|
417
|
+
});
|
|
418
|
+
});
|
|
@@ -9,6 +9,43 @@ import { Queue } from "./queue";
|
|
|
9
9
|
import { Binding, Event } from "./types";
|
|
10
10
|
import { metricsCollector } from "./metrics";
|
|
11
11
|
|
|
12
|
+
// stores Queue instances
|
|
13
|
+
const queues: Record<string, Queue<KubernetesObject>> = {};
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Get the key for an entry in the queues
|
|
17
|
+
*
|
|
18
|
+
* @param obj The object to derive a key from
|
|
19
|
+
* @returns The key to a Queue in the list of queues
|
|
20
|
+
*/
|
|
21
|
+
export function queueKey(obj: KubernetesObject) {
|
|
22
|
+
const options = ["kind", "kindNs", "kindNsName", "global"];
|
|
23
|
+
const d3fault = "kind";
|
|
24
|
+
|
|
25
|
+
let strat = process.env.PEPR_RECONCILE_STRATEGY || d3fault;
|
|
26
|
+
strat = options.includes(strat) ? strat : d3fault;
|
|
27
|
+
|
|
28
|
+
const ns = obj.metadata?.namespace ?? "cluster-scoped";
|
|
29
|
+
const kind = obj.kind ?? "UnknownKind";
|
|
30
|
+
const name = obj.metadata?.name ?? "Unnamed";
|
|
31
|
+
|
|
32
|
+
const lookup: Record<string, string> = {
|
|
33
|
+
kind: `${kind}`,
|
|
34
|
+
kindNs: `${kind}/${ns}`,
|
|
35
|
+
kindNsName: `${kind}/${ns}/${name}`,
|
|
36
|
+
global: "global",
|
|
37
|
+
};
|
|
38
|
+
return lookup[strat];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function getOrCreateQueue(obj: KubernetesObject) {
|
|
42
|
+
const key = queueKey(obj);
|
|
43
|
+
if (!queues[key]) {
|
|
44
|
+
queues[key] = new Queue<KubernetesObject>(key);
|
|
45
|
+
}
|
|
46
|
+
return queues[key];
|
|
47
|
+
}
|
|
48
|
+
|
|
12
49
|
// Watch configuration
|
|
13
50
|
const watchCfg: WatchCfg = {
|
|
14
51
|
resyncFailureMax: process.env.PEPR_RESYNC_FAILURE_MAX ? parseInt(process.env.PEPR_RESYNC_FAILURE_MAX, 10) : 5,
|
|
@@ -18,7 +55,7 @@ const watchCfg: WatchCfg = {
|
|
|
18
55
|
: 300,
|
|
19
56
|
relistIntervalSec: process.env.PEPR_RELIST_INTERVAL_SECONDS
|
|
20
57
|
? parseInt(process.env.PEPR_RELIST_INTERVAL_SECONDS, 10)
|
|
21
|
-
:
|
|
58
|
+
: 600,
|
|
22
59
|
};
|
|
23
60
|
|
|
24
61
|
// Map the event to the watch phase
|
|
@@ -54,16 +91,16 @@ async function runBinding(binding: Binding, capabilityNamespaces: string[]) {
|
|
|
54
91
|
const phaseMatch: WatchPhase[] = eventToPhaseMap[binding.event] || eventToPhaseMap[Event.Any];
|
|
55
92
|
|
|
56
93
|
// The watch callback is run when an object is received or dequeued
|
|
57
|
-
|
|
58
94
|
Log.debug({ watchCfg }, "Effective WatchConfig");
|
|
59
|
-
|
|
95
|
+
|
|
96
|
+
const watchCallback = async (obj: KubernetesObject, phase: WatchPhase) => {
|
|
60
97
|
// First, filter the object based on the phase
|
|
61
|
-
if (phaseMatch.includes(
|
|
98
|
+
if (phaseMatch.includes(phase)) {
|
|
62
99
|
try {
|
|
63
100
|
// Then, check if the object matches the filter
|
|
64
101
|
const filterMatch = filterNoMatchReason(binding, obj, capabilityNamespaces);
|
|
65
102
|
if (filterMatch === "") {
|
|
66
|
-
await binding.watchCallback?.(obj,
|
|
103
|
+
await binding.watchCallback?.(obj, phase);
|
|
67
104
|
} else {
|
|
68
105
|
Log.debug(filterMatch);
|
|
69
106
|
}
|
|
@@ -74,19 +111,15 @@ async function runBinding(binding: Binding, capabilityNamespaces: string[]) {
|
|
|
74
111
|
}
|
|
75
112
|
};
|
|
76
113
|
|
|
77
|
-
const queue = new Queue();
|
|
78
|
-
queue.setReconcile(watchCallback);
|
|
79
|
-
|
|
80
114
|
// Setup the resource watch
|
|
81
|
-
const watcher = K8s(binding.model, binding.filters).Watch(async (obj,
|
|
82
|
-
Log.debug(obj, `Watch event ${
|
|
115
|
+
const watcher = K8s(binding.model, binding.filters).Watch(async (obj, phase) => {
|
|
116
|
+
Log.debug(obj, `Watch event ${phase} received`);
|
|
83
117
|
|
|
84
|
-
// If the binding is a queue, enqueue the object
|
|
85
118
|
if (binding.isQueue) {
|
|
86
|
-
|
|
119
|
+
const queue = getOrCreateQueue(obj);
|
|
120
|
+
await queue.enqueue(obj, phase, watchCallback);
|
|
87
121
|
} else {
|
|
88
|
-
|
|
89
|
-
await watchCallback(obj, type);
|
|
122
|
+
await watchCallback(obj, phase);
|
|
90
123
|
}
|
|
91
124
|
}, watchCfg);
|
|
92
125
|
|
|
@@ -130,8 +163,8 @@ async function runBinding(binding: Binding, capabilityNamespaces: string[]) {
|
|
|
130
163
|
}
|
|
131
164
|
}
|
|
132
165
|
|
|
133
|
-
export function logEvent(
|
|
134
|
-
const logMessage = `Watch event ${
|
|
166
|
+
export function logEvent(event: WatchEvent, message: string = "", obj?: KubernetesObject) {
|
|
167
|
+
const logMessage = `Watch event ${event} received${message ? `. ${message}.` : "."}`;
|
|
135
168
|
if (obj) {
|
|
136
169
|
Log.debug(obj, logMessage);
|
|
137
170
|
} else {
|
package/src/sdk/sdk.test.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { beforeEach, describe, it, jest } from "@jest/globals";
|
|
|
11
11
|
import { GenericKind } from "kubernetes-fluent-client";
|
|
12
12
|
import { K8s, kind } from "kubernetes-fluent-client";
|
|
13
13
|
import { Mock } from "jest-mock";
|
|
14
|
+
import { V1OwnerReference } from "@kubernetes/client-node";
|
|
14
15
|
|
|
15
16
|
jest.mock("kubernetes-fluent-client", () => ({
|
|
16
17
|
K8s: jest.fn(),
|
|
@@ -163,23 +164,55 @@ describe("writeEvent", () => {
|
|
|
163
164
|
});
|
|
164
165
|
|
|
165
166
|
describe("getOwnerRefFrom", () => {
|
|
166
|
-
|
|
167
|
-
|
|
167
|
+
const customResource = {
|
|
168
|
+
apiVersion: "v1",
|
|
169
|
+
kind: "Package",
|
|
170
|
+
metadata: { name: "test", namespace: "default", uid: "1" },
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
const ownerRef = [
|
|
174
|
+
{
|
|
168
175
|
apiVersion: "v1",
|
|
169
176
|
kind: "Package",
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
177
|
+
name: "test",
|
|
178
|
+
uid: "1",
|
|
179
|
+
},
|
|
180
|
+
];
|
|
181
|
+
|
|
182
|
+
const ownerRefWithController = ownerRef.map(item => ({
|
|
183
|
+
...item,
|
|
184
|
+
controller: true,
|
|
185
|
+
}));
|
|
186
|
+
const ownerRefWithBlockOwnerDeletion = ownerRef.map(item => ({
|
|
187
|
+
...item,
|
|
188
|
+
blockOwnerDeletion: false,
|
|
189
|
+
}));
|
|
190
|
+
const ownerRefWithAllFields = ownerRef.map(item => ({
|
|
191
|
+
...item,
|
|
192
|
+
blockOwnerDeletion: true,
|
|
193
|
+
controller: false,
|
|
194
|
+
}));
|
|
195
|
+
|
|
196
|
+
test.each([
|
|
197
|
+
[true, false, ownerRefWithAllFields],
|
|
198
|
+
[false, undefined, ownerRefWithBlockOwnerDeletion],
|
|
199
|
+
[undefined, true, ownerRefWithController],
|
|
200
|
+
[undefined, undefined, ownerRef],
|
|
201
|
+
])(
|
|
202
|
+
"should return owner reference for the CRD for combinations of V1OwnerReference fields - Optionals: blockOwnerDeletion (%s), controller (%s)",
|
|
203
|
+
(blockOwnerDeletion, controller, expected) => {
|
|
204
|
+
const result = getOwnerRefFrom(customResource, blockOwnerDeletion, controller);
|
|
205
|
+
expect(result).toStrictEqual(expected);
|
|
206
|
+
},
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
it("should support all defined fields in the V1OwnerReference type", () => {
|
|
210
|
+
const V1OwnerReferenceFieldCount = Object.getOwnPropertyNames(V1OwnerReference).length;
|
|
211
|
+
const result = getOwnerRefFrom(customResource, false, true);
|
|
212
|
+
expect(Object.keys(result[0]).length).toEqual(V1OwnerReferenceFieldCount);
|
|
181
213
|
});
|
|
182
214
|
});
|
|
215
|
+
|
|
183
216
|
describe("sanitizeResourceName Fuzzing Tests", () => {
|
|
184
217
|
test("should handle any random string input", () => {
|
|
185
218
|
fc.assert(
|
package/src/sdk/sdk.ts
CHANGED
|
@@ -79,18 +79,27 @@ export async function writeEvent(
|
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
81
|
* Get the owner reference for a custom resource
|
|
82
|
-
* @param
|
|
83
|
-
* @
|
|
82
|
+
* @param customResource the custom resource to get the owner reference for
|
|
83
|
+
* @param blockOwnerDeletion if true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed.
|
|
84
|
+
* @param controller if true, this reference points to the managing controller.
|
|
85
|
+
* @returns the owner reference array for the custom resource
|
|
84
86
|
*/
|
|
85
|
-
export function getOwnerRefFrom(
|
|
86
|
-
|
|
87
|
+
export function getOwnerRefFrom(
|
|
88
|
+
customResource: GenericKind,
|
|
89
|
+
blockOwnerDeletion?: boolean,
|
|
90
|
+
controller?: boolean,
|
|
91
|
+
): V1OwnerReference[] {
|
|
92
|
+
const { apiVersion, kind, metadata } = customResource;
|
|
93
|
+
const { name, uid } = metadata!;
|
|
87
94
|
|
|
88
95
|
return [
|
|
89
96
|
{
|
|
90
|
-
apiVersion:
|
|
91
|
-
kind:
|
|
97
|
+
apiVersion: apiVersion!,
|
|
98
|
+
kind: kind!,
|
|
92
99
|
uid: uid!,
|
|
93
100
|
name: name!,
|
|
101
|
+
...(blockOwnerDeletion !== undefined && { blockOwnerDeletion }),
|
|
102
|
+
...(controller !== undefined && { controller }),
|
|
94
103
|
},
|
|
95
104
|
];
|
|
96
105
|
}
|