pepr 0.42.0 → 0.42.2

package/src/lib/controller/store.ts

@@ -5,14 +5,15 @@ import { Operation } from "fast-json-patch";
 import { K8s } from "kubernetes-fluent-client";
 import { startsWith } from "ramda";
 
-import { Capability } from "../capability";
+import { Capability } from "../core/capability";
 import { Store } from "../k8s";
 import Log, { redactedPatch, redactedStore } from "../telemetry/logger";
-import { DataOp, DataSender, DataStore, Storage } from "../storage";
+import { DataOp, DataSender, DataStore, Storage } from "../core/storage";
 import { fillStoreCache, sendUpdatesAndFlushCache } from "./storeCache";
 
 const namespace = "pepr-system";
-export const debounceBackoff = 1000;
+const debounceBackoffReceive = 1000;
+const debounceBackoffSend = 4000;
 
 export class StoreController {
   #name: string;
@@ -25,7 +26,7 @@ export class StoreController {
 
     this.#name = name;
 
-    const setStorageInstance = (registrationFunction: () => Storage, name: string) => {
+    const setStorageInstance = (registrationFunction: () => Storage, name: string): void => {
       const scheduleStore = registrationFunction();
 
       // Bind the store sender to the capability
@@ -61,13 +62,22 @@ export class StoreController {
     );
   }
 
-  #setupWatch = () => {
+  #setupWatch = (): void => {
     const watcher = K8s(Store, { name: this.#name, namespace }).Watch(this.#receive);
     watcher.start().catch(e => Log.error(e, "Error starting Pepr store watch"));
   };
 
-  #migrateAndSetupWatch = async (store: Store) => {
+  #migrateAndSetupWatch = async (store: Store): Promise<void> => {
     Log.debug(redactedStore(store), "Pepr Store migration");
+    // Add cacheID label to store
+    await K8s(Store, { namespace, name: this.#name }).Patch([
+      {
+        op: "add",
+        path: "/metadata/labels/pepr.dev-cacheID",
+        value: `${Date.now()}`,
+      },
+    ]);
+
     const data: DataStore = store.data || {};
     let storeCache: Record<string, Operation> = {};
 
@@ -96,11 +106,11 @@ export class StoreController {
     this.#setupWatch();
   };
 
-  #receive = (store: Store) => {
+  #receive = (store: Store): void => {
     Log.debug(redactedStore(store), "Pepr Store update");
 
     // Wrap the update in a debounced function
-    const debounced = () => {
+    const debounced = (): void => {
       // Base64 decode the data
       const data: DataStore = store.data || {};
 
@@ -134,10 +144,10 @@ export class StoreController {
 
     // Debounce the update to 1 second to avoid multiple rapid calls
     clearTimeout(this.#sendDebounce);
-    this.#sendDebounce = setTimeout(debounced, this.#onReady ? 0 : debounceBackoff);
+    this.#sendDebounce = setTimeout(debounced, this.#onReady ? 0 : debounceBackoffReceive);
   };
 
-  #send = (capabilityName: string) => {
+  #send = (capabilityName: string): DataSender => {
     let storeCache: Record<string, Operation> = {};
 
     // Create a sender function for the capability to add/remove data from the store
@@ -151,12 +161,12 @@ export class StoreController {
         Log.debug(redactedPatch(storeCache), "Sending updates to Pepr store");
         void sendUpdatesAndFlushCache(storeCache, namespace, this.#name);
       }
-    }, debounceBackoff);
+    }, debounceBackoffSend);
 
     return sender;
   };
 
-  #createStoreResource = async (e: unknown) => {
+  #createStoreResource = async (e: unknown): Promise<void> => {
     Log.info(`Pepr store not found, creating...`);
     Log.debug(e);
 
@@ -165,6 +175,9 @@ export class StoreController {
         metadata: {
           name: this.#name,
           namespace,
+          labels: {
+            "pepr.dev-cacheID": `${Date.now()}`,
+          },
         },
         data: {
           // JSON Patch will die if the data is empty, so we need to add a placeholder

package/src/lib/controller/storeCache.ts

@@ -1,17 +1,21 @@
-import { DataOp } from "../storage";
+import { DataOp } from "../core/storage";
 import Log from "../telemetry/logger";
 import { K8s } from "kubernetes-fluent-client";
 import { Store } from "../k8s";
 import { StatusCodes } from "http-status-codes";
 import { Operation } from "fast-json-patch";
 
-export const sendUpdatesAndFlushCache = async (cache: Record<string, Operation>, namespace: string, name: string) => {
+export const sendUpdatesAndFlushCache = async (
+  cache: Record<string, Operation>,
+  namespace: string,
+  name: string,
+): Promise<Record<string, Operation>> => {
   const indexes = Object.keys(cache);
   const payload = Object.values(cache);
 
   try {
     if (payload.length > 0) {
-      await K8s(Store, { namespace, name }).Patch(payload); // Send patch to cluster
+      await K8s(Store, { namespace, name }).Patch(updateCacheID(payload)); // Send patch to cluster
       Object.keys(cache).forEach(key => delete cache[key]);
     }
   } catch (err) {
@@ -61,3 +65,12 @@ export const fillStoreCache = (
   }
   return cache;
 };
+
+export function updateCacheID(payload: Operation[]): Operation[] {
+  payload.push({
+    op: "replace",
+    path: "/metadata/labels/pepr.dev-cacheID",
+    value: `${Date.now()}`,
+  });
+  return payload;
+}

package/src/lib/{capability.ts → core/capability.ts}

@@ -3,11 +3,11 @@
 
 import { GenericClass, GroupVersionKind, modelToGroupVersionKind } from "kubernetes-fluent-client";
 import { pickBy } from "ramda";
-import Log from "./telemetry/logger";
+import Log from "../telemetry/logger";
 import { isBuildMode, isDevMode, isWatchMode } from "./module";
 import { PeprStore, Storage } from "./storage";
 import { OnSchedule, Schedule } from "./schedule";
-import { Event } from "./enums";
+import { Event } from "../enums";
 import {
   Binding,
   BindingFilter,
@@ -22,8 +22,8 @@ import {
   FinalizeAction,
   FinalizeActionChain,
   WhenSelector,
-} from "./types";
-import { addFinalizer } from "./finalizer";
+} from "../types";
+import { addFinalizer } from "../finalizer";
 
 const registerAdmission = isBuildMode() || !isWatchMode();
 const registerWatch = isBuildMode() || isWatchMode() || isDevMode();
@@ -71,7 +71,7 @@ export class Capability implements CapabilityExport {
     }
   };
 
-  public getScheduleStore() {
+  public getScheduleStore(): Storage {
     return this.#scheduleStore;
   }
 
@@ -111,19 +111,19 @@ export class Capability implements CapabilityExport {
     onReady: this.#scheduleStore.onReady,
   };
 
-  get bindings() {
+  get bindings(): Binding[] {
     return this.#bindings;
   }
 
-  get name() {
+  get name(): string {
     return this.#name;
   }
 
-  get description() {
+  get description(): string {
     return this.#description;
   }
 
-  get namespaces() {
+  get namespaces(): string[] {
     return this.#namespaces || [];
   }
 
@@ -207,8 +207,19 @@ export class Capability implements CapabilityExport {
     const bindings = this.#bindings;
     const prefix = `${this.#name}: ${model.name}`;
     const commonChain = { WithLabel, WithAnnotation, WithDeletionTimestamp, Mutate, Validate, Watch, Reconcile, Alias };
-    const isNotEmpty = (value: object) => Object.keys(value).length > 0;
-    const log = (message: string, cbString: string) => {
+
+    type CommonChainType = typeof commonChain;
+    type ExtendedCommonChainType = CommonChainType & {
+      Alias: (alias: string) => CommonChainType;
+      InNamespace: (...namespaces: string[]) => BindingWithName<T>;
+      InNamespaceRegex: (...namespaces: RegExp[]) => BindingWithName<T>;
+      WithName: (name: string) => BindingFilter<T>;
+      WithNameRegex: (regexName: RegExp) => BindingFilter<T>;
+      WithDeletionTimestamp: () => BindingFilter<T>;
+    };
+
+    const isNotEmpty = (value: object): boolean => Object.keys(value).length > 0;
+    const log = (message: string, cbString: string): void => {
       const filteredObj = pickBy(isNotEmpty, binding.filters);
 
       Log.info(`${message} configured for ${binding.event}`, prefix);
@@ -329,7 +340,7 @@ export class Capability implements CapabilityExport {
           isWatch: true,
           isFinalize: true,
           event: Event.UPDATE,
-          finalizeCallback: async (update: InstanceType<T>, logger = aliasLogger) => {
+          finalizeCallback: async (update: InstanceType<T>, logger = aliasLogger): Promise<boolean | void> => {
             Log.info(`Executing finalize action with alias: ${binding.alias || "no alias provided"}`);
             return await finalizeCallback(update, logger);
           },
@@ -380,13 +391,13 @@ export class Capability implements CapabilityExport {
       return commonChain;
     }
 
-    function Alias(alias: string) {
+    function Alias(alias: string): CommonChainType {
       Log.debug(`Adding prefix alias ${alias}`, prefix);
       binding.alias = alias;
       return commonChain;
     }
 
-    function bindEvent(event: Event) {
+    function bindEvent(event: Event): ExtendedCommonChainType {
       binding.event = event;
       return {
         ...commonChain,

package/src/lib/{module.ts → core/module.ts}

@@ -2,12 +2,12 @@
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 import { clone } from "ramda";
 import { Capability } from "./capability";
-import { Controller } from "./controller";
-import { ValidateError } from "./errors";
-import { MutateResponse, ValidateResponse, WebhookIgnore } from "./k8s";
-import { CapabilityExport, AdmissionRequest } from "./types";
-import { setupWatch } from "./watch-processor";
-import { Log } from "../lib";
+import { Controller } from "../controller";
+import { ValidateError } from "../errors";
+import { MutateResponse, ValidateResponse, WebhookIgnore } from "../k8s";
+import { CapabilityExport, AdmissionRequest } from "../types";
+import { setupWatch } from "../processors/watch-processor";
+import { Log } from "../../lib";
 import { V1PolicyRule as PolicyRule } from "@kubernetes/client-node";
 
 /** Custom Labels Type for package.json */
@@ -58,12 +58,12 @@ export type PeprModuleOptions = {
 };
 
 // Track if this is a watch mode controller
-export const isWatchMode = () => process.env.PEPR_WATCH_MODE === "true";
+export const isWatchMode = (): boolean => process.env.PEPR_WATCH_MODE === "true";
 
 // Track if Pepr is running in build mode
-export const isBuildMode = () => process.env.PEPR_MODE === "build";
+export const isBuildMode = (): boolean => process.env.PEPR_MODE === "build";
 
-export const isDevMode = () => process.env.PEPR_MODE === "dev";
+export const isDevMode = (): boolean => process.env.PEPR_MODE === "dev";
 
 export class PeprModule {
   #controller!: Controller;
@@ -135,7 +135,7 @@ export class PeprModule {
    *
    * @param port
    */
-  start = (port = 3000) => {
+  start = (port = 3000): void => {
     this.#controller.startServer(port);
   };
 }

package/src/lib/{queue.ts → core/queue.ts}

@@ -3,7 +3,7 @@
 import { KubernetesObject } from "@kubernetes/client-node";
 import { WatchPhase } from "kubernetes-fluent-client/dist/fluent/types";
 import { randomBytes } from "node:crypto";
-import Log from "./telemetry/logger";
+import Log from "../telemetry/logger";
 
 type WatchCallback = (obj: KubernetesObject, phase: WatchPhase) => Promise<void>;
 
@@ -29,11 +29,19 @@ export class Queue<K extends KubernetesObject> {
     this.#uid = `${Date.now()}-${randomBytes(2).toString("hex")}`;
   }
 
-  label() {
+  label(): { name: string; uid: string } {
     return { name: this.#name, uid: this.#uid };
   }
 
-  stats() {
+  stats(): {
+    queue: {
+      name: string;
+      uid: string;
+    };
+    stats: {
+      length: number;
+    };
+  } {
     return {
       queue: this.label(),
       stats: {
@@ -51,7 +59,7 @@ export class Queue<K extends KubernetesObject> {
    * @param reconcile The callback to enqueue for reconcile
    * @returns A promise that resolves when the object is reconciled
    */
-  enqueue(item: K, phase: WatchPhase, reconcile: WatchCallback) {
+  enqueue(item: K, phase: WatchPhase, reconcile: WatchCallback): Promise<void> {
     const note = {
       queue: this.label(),
       item: {
@@ -73,7 +81,7 @@ export class Queue<K extends KubernetesObject> {
    *
    * @returns A promise that resolves when the webapp is reconciled
    */
-  async #dequeue() {
+  async #dequeue(): Promise<false | undefined> {
     // If there is a pending promise, do nothing
     if (this.#pendingPromise) {
       Log.debug("Pending promise, not dequeuing");

package/src/lib/{storage.ts → core/storage.ts}

@@ -12,6 +12,11 @@ export type Unsubscribe = () => void;
 const MAX_WAIT_TIME = 15000;
 const STORE_VERSION_PREFIX = "v2";
 
+interface WaitRecord {
+  timeout?: ReturnType<typeof setTimeout>;
+  unsubscribe?: () => void;
+}
+
 export function v2StoreKey(key: string): string {
   return `${STORE_VERSION_PREFIX}-${pointer.escape(key)}`;
 }
@@ -58,13 +63,13 @@ export interface PeprStore {
    * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
    * Resolves when the key/value show up in the store.
    */
-  setItemAndWait(key: string, value: string): Promise<void>;
+  setItemAndWait(key: string, value: string): Promise<string>;
 
   /**
    * Remove the value of the key.
    * Resolves when the key does not show up in the store.
    */
-  removeItemAndWait(key: string): Promise<void>;
+  removeItemAndWait(key: string): Promise<string>;
 }
 
 /**
@@ -128,22 +133,24 @@ export class Storage implements PeprStore {
    * @param value - The value of the key
    * @returns
    */
-  setItemAndWait = (key: string, value: string): Promise<void> => {
+  setItemAndWait = (key: string, value: string): Promise<string> => {
     this.#dispatchUpdate("add", [v2StoreKey(key)], value);
+    const record: WaitRecord = {};
 
-    return new Promise<void>((resolve, reject) => {
-      const unsubscribe = this.subscribe(data => {
+    return new Promise<string>((resolve, reject) => {
+      // If promise has not resolved before MAX_WAIT_TIME reject
+      record.timeout = setTimeout(() => {
+        record.unsubscribe!();
+        return reject(`MAX_WAIT_TIME elapsed: Key ${key} not seen in ${MAX_WAIT_TIME / 1000}s`);
+      }, MAX_WAIT_TIME);
+
+      record.unsubscribe = this.subscribe(data => {
         if (data[`${v2UnescapedStoreKey(key)}`] === value) {
-          unsubscribe();
-          resolve();
+          record.unsubscribe!();
+          clearTimeout(record.timeout);
+          resolve("ok");
         }
       });
-
-      // If promise has not resolved before MAX_WAIT_TIME reject
-      setTimeout(() => {
-        unsubscribe();
-        return reject();
-      }, MAX_WAIT_TIME);
     });
   };
 
@@ -154,21 +161,23 @@ export class Storage implements PeprStore {
    * @param key - The key to add into the store
    * @returns
    */
-  removeItemAndWait = (key: string): Promise<void> => {
+  removeItemAndWait = (key: string): Promise<string> => {
     this.#dispatchUpdate("remove", [v2StoreKey(key)]);
-    return new Promise<void>((resolve, reject) => {
-      const unsubscribe = this.subscribe(data => {
+    const record: WaitRecord = {};
+    return new Promise<string>((resolve, reject) => {
+      // If promise has not resolved before MAX_WAIT_TIME reject
+      record.timeout = setTimeout(() => {
+        record.unsubscribe!();
+        return reject(`MAX_WAIT_TIME elapsed: Key ${key} still seen after ${MAX_WAIT_TIME / 1000}s`);
+      }, MAX_WAIT_TIME);
+
+      record.unsubscribe = this.subscribe(data => {
         if (!Object.hasOwn(data, `${v2UnescapedStoreKey(key)}`)) {
-          unsubscribe();
-          resolve();
+          record.unsubscribe!();
+          clearTimeout(record.timeout);
+          resolve("ok");
         }
       });
-
-      // If promise has not resolved before MAX_WAIT_TIME reject
-      setTimeout(() => {
-        unsubscribe();
-        return reject();
-      }, MAX_WAIT_TIME);
     });
   };
 

package/src/lib/deploymentChecks.ts

@@ -4,7 +4,7 @@ import { K8s, kind } from "kubernetes-fluent-client";
 import Log from "./telemetry/logger";
 
 // returns true if all deployments are ready, false otherwise
-export async function checkDeploymentStatus(namespace: string) {
+export async function checkDeploymentStatus(namespace: string): Promise<boolean> {
   const deployments = await K8s(kind.Deployment).InNamespace(namespace).Get();
   let status = false;
   let readyCount = 0;
@@ -29,7 +29,7 @@ export async function checkDeploymentStatus(namespace: string) {
 }
 
 // wait for all deployments in the pepr-system namespace to be ready
-export async function namespaceDeploymentsReady(namespace: string = "pepr-system") {
+export async function namespaceDeploymentsReady(namespace: string = "pepr-system"): Promise<true | undefined> {
   Log.info(`Checking ${namespace} deployments status...`);
   let ready = false;
   while (!ready) {

package/src/lib/errors.ts

@@ -1,19 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 
-export const Errors = {
-  audit: "audit",
-  ignore: "ignore",
-  reject: "reject",
-};
-
-export const ErrorList = Object.values(Errors);
+import { OnError } from "../cli/init/enums";
 
+export const ErrorList = Object.values(OnError) as string[];
 /**
  * Validate the error or throw an error
  * @param error
  */
-export function ValidateError(error = "") {
+export function ValidateError(error: string = ""): void {
   if (!ErrorList.includes(error)) {
     throw new Error(`Invalid error: ${error}. Must be one of: ${ErrorList.join(", ")}`);
   }

package/src/lib/filesystemService.ts

@@ -3,7 +3,7 @@
 
 import { promises } from "fs";
 
-export async function createDirectoryIfNotExists(path: string) {
+export async function createDirectoryIfNotExists(path: string): Promise<void> {
   try {
     await promises.access(path);
   } catch (error) {

package/src/lib/filter/adjudicators/adjudicators.ts

@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 
 import { Event, Operation } from "../../enums";
-import { AdmissionRequest, Binding } from "../../types";
+import { AdmissionRequest, Binding, FinalizeAction, WatchLogAction, MutateAction, ValidateAction } from "../../types";
 import {
   __,
   allPass,
@@ -19,7 +19,7 @@ import {
   nthArg,
   pipe,
 } from "ramda";
-import { KubernetesObject } from "kubernetes-fluent-client";
+import { GenericClass, KubernetesObject } from "kubernetes-fluent-client";
 
 /*
   Naming scheme:
@@ -77,6 +77,7 @@ export const carriedNamespace = pipe(
   (kubernetesObject: KubernetesObject): string | undefined => kubernetesObject?.metadata?.namespace,
   defaultTo(""),
 );
+
 export const carriesNamespace = pipe(carriedNamespace, equals(""), not);
 
 export const carriedAnnotations = pipe(
@@ -144,7 +145,7 @@ export const definesVersion = pipe(definedVersion, equals(""), not);
 export const definedKind = pipe((binding): string => binding?.kind?.kind, defaultTo(""));
 export const definesKind = pipe(definedKind, equals(""), not);
 
-export const definedCategory = (binding: Partial<Binding>) => {
+export const definedCategory = (binding: Partial<Binding>): string => {
   // Ordering matters, finalize is a "watch"
   // prettier-ignore
   return binding.isFinalize ? "Finalize" :
@@ -153,8 +154,15 @@ export const definedCategory = (binding: Partial<Binding>) => {
     binding.isValidate ? "Validate" :
     "";
 };
-
-export const definedCallback = (binding: Partial<Binding>) => {
+export type DefinedCallbackReturnType =
+  | FinalizeAction<GenericClass, InstanceType<GenericClass>>
+  | WatchLogAction<GenericClass, InstanceType<GenericClass>>
+  | MutateAction<GenericClass, InstanceType<GenericClass>>
+  | ValidateAction<GenericClass, InstanceType<GenericClass>>
+  | null
+  | undefined;
+
+export const definedCallback = (binding: Partial<Binding>): DefinedCallbackReturnType => {
   // Ordering matters, finalize is a "watch"
   // prettier-ignore
   return binding.isFinalize ? binding.finalizeCallback :
@@ -241,10 +249,21 @@ export const mismatchedLabels = allPass([
   pipe((binding, kubernetesObject) => metasMismatch(definedLabels(binding), carriedLabels(kubernetesObject))),
 ]);
 
+/*
+ * If the object does not have a namespace, and it is not a namespace,
+ * then we must return false because it cannot be uncarryable
+ */
 export const uncarryableNamespace = allPass([
   pipe(nthArg(0), length, gt(__, 0)),
-  pipe(nthArg(1), carriesNamespace),
-  pipe((namespaceSelector, kubernetesObject) => namespaceSelector.includes(carriedNamespace(kubernetesObject)), not),
+  pipe((namespaceSelector, kubernetesObject) => {
+    if (kubernetesObject?.kind === "Namespace") {
+      return namespaceSelector.includes(kubernetesObject?.metadata?.name);
+    }
+    if (carriesNamespace(kubernetesObject)) {
+      return namespaceSelector.includes(carriedNamespace(kubernetesObject));
+    }
+    return true;
+  }, not),
 ]);
 
 export const missingCarriableNamespace = allPass([
@@ -256,10 +275,22 @@ export const missingCarriableNamespace = allPass([
   ),
 ]);
 
+/*
+ * If the object does not have a namespace, and it is not a namespace,
+ * then we must return false because it cannot be ignored
+ */
 export const carriesIgnoredNamespace = allPass([
   pipe(nthArg(0), length, gt(__, 0)),
-  pipe(nthArg(1), carriesNamespace),
-  pipe((namespaceSelector, kubernetesObject) => namespaceSelector.includes(carriedNamespace(kubernetesObject))),
+  pipe((namespaceSelector, kubernetesObject) => {
+    if (kubernetesObject?.kind === "Namespace") {
+      return namespaceSelector.includes(kubernetesObject?.metadata?.name);
+    }
+    if (carriesNamespace(kubernetesObject)) {
+      return namespaceSelector.includes(carriedNamespace(kubernetesObject));
+    }
+
+    return false;
+  }),
 ]);
 
 export const unbindableNamespaces = allPass([