pepr 0.13.4 → 0.14.0

package/src/lib/assets/webhooks.ts

@@ -6,10 +6,10 @@ import {
   V1LabelSelectorRequirement,
   V1RuleWithOperations,
 } from "@kubernetes/client-node";
+import { kind } from "kubernetes-fluent-client";
 import { concat, equals, uniqWith } from "ramda";
 
 import { Assets } from ".";
-import { MutatingWebhookConfiguration, ValidatingWebhookConfiguration } from "../k8s/upstream";
 import { Event } from "../types";
 
 const peprIgnoreLabel: V1LabelSelectorRequirement = {
@@ -70,7 +70,7 @@ export async function webhookConfig(
   assets: Assets,
   mutateOrValidate: "mutate" | "validate",
   timeoutSeconds = 10,
-): Promise<MutatingWebhookConfiguration | ValidatingWebhookConfiguration | null> {
+): Promise<kind.MutatingWebhookConfiguration | kind.ValidatingWebhookConfiguration | null> {
   const ignore = [peprIgnoreLabel];
 
   const { name, tls, config, apiToken, host } = assets;

package/src/lib/assets/yaml.ts

@@ -6,10 +6,11 @@ import crypto from "crypto";
 import { promises as fs } from "fs";
 
 import { Assets } from ".";
-import { apiTokenSecret, service, tlsSecret } from "./networking";
-import { deployment, moduleSecret, namespace } from "./pods";
-import { clusterRole, clusterRoleBinding, serviceAccount } from "./rbac";
+import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking";
+import { deployment, moduleSecret, namespace, watcher } from "./pods";
+import { clusterRole, clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac";
 import { webhookConfig } from "./webhooks";
+import { peprStoreCRD } from "./store";
 
 export function zarfYaml({ name, image, config }: Assets, path: string) {
   const zarfCfg = {
@@ -49,6 +50,7 @@ export async function allYaml(assets: Assets) {
 
   const mutateWebhook = await webhookConfig(assets, "mutate");
   const validateWebhook = await webhookConfig(assets, "validate");
+  const watchDeployment = watcher(assets, hash);
 
   const resources = [
     namespace,
@@ -59,7 +61,11 @@ export async function allYaml(assets: Assets) {
     tlsSecret(name, tls),
     deployment(assets, hash),
     service(name),
+    watcherService(name),
     moduleSecret(name, code, hash),
+    peprStoreCRD,
+    storeRole(name),
+    storeRoleBinding(name),
   ];
 
   if (mutateWebhook) {
@@ -70,6 +76,10 @@ export async function allYaml(assets: Assets) {
     resources.push(validateWebhook);
   }
 
+  if (watchDeployment) {
+    resources.push(watchDeployment);
+  }
+
   // Convert the resources to a single YAML string
   return resources.map(r => dumpYaml(r, { noRefs: true })).join("---\n");
 }

package/src/lib/capability.ts

@@ -1,11 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 
+import { GenericClass, GroupVersionKind, modelToGroupVersionKind } from "kubernetes-fluent-client";
+import { WatchAction } from "kubernetes-fluent-client/dist/fluent/types";
 import { pickBy } from "ramda";
 
-import { isWatchMode, modelToGroupVersionKind } from "./k8s/index";
-import { GroupVersionKind } from "./k8s/types";
 import Log from "./logger";
+import { isBuildMode, isDevMode, isWatchMode } from "./module";
+import { PeprStore, Storage } from "./storage";
 import {
   Binding,
   BindingFilter,
@@ -13,13 +15,16 @@ import {
   CapabilityCfg,
   CapabilityExport,
   Event,
-  GenericClass,
   MutateAction,
   MutateActionChain,
   ValidateAction,
+  ValidateActionChain,
   WhenSelector,
 } from "./types";
 
+const registerAdmission = isBuildMode() || !isWatchMode();
+const registerWatch = isBuildMode() || isWatchMode() || isDevMode();
+
 /**
  * A capability is a unit of functionality that can be registered with the Pepr runtime.
  */
@@ -28,6 +33,24 @@ export class Capability implements CapabilityExport {
   #description: string;
   #namespaces?: string[] | undefined;
   #bindings: Binding[] = [];
+  #store = new Storage();
+  #registered = false;
+
+  /**
+   * Store is a key-value data store that can be used to persist data that should be shared
+   * between requests. Each capability has its own store, and the data is persisted in Kubernetes
+   * in the `pepr-system` namespace.
+   *
+   * Note: You should only access the store from within an action.
+   */
+  Store: PeprStore = {
+    clear: this.#store.clear,
+    getItem: this.#store.getItem,
+    removeItem: this.#store.removeItem,
+    setItem: this.#store.setItem,
+    subscribe: this.#store.subscribe,
+    onReady: this.#store.onReady,
+  };
 
   get bindings() {
     return this.#bindings;
@@ -50,15 +73,32 @@ export class Capability implements CapabilityExport {
     this.#description = cfg.description;
     this.#namespaces = cfg.namespaces;
 
-    // Bind When() to this instance
-    this.When = this.When.bind(this);
-
     Log.info(`Capability ${this.#name} registered`);
     Log.debug(cfg);
   }
 
   /**
-   * The When method is used to register a capability action to be executed when a Kubernetes resource is
+   * Register the store with the capability. This is called automatically by the Pepr controller.
+   *
+   * @param store
+   */
+  registerStore = () => {
+    Log.info(`Registering store for ${this.#name}`);
+
+    if (this.#registered) {
+      throw new Error(`Store already registered for ${this.#name}`);
+    }
+
+    this.#registered = true;
+
+    // Pass back any ready callback to the controller
+    return {
+      store: this.#store,
+    };
+  };
+
+  /**
+   * The When method is used to register a action to be executed when a Kubernetes resource is
    * processed by Pepr. The action will be executed if the resource matches the specified kind and any
    * filters that are applied.
    *
@@ -66,7 +106,7 @@ export class Capability implements CapabilityExport {
    * @param kind if using a custom KubernetesObject not available in `a.*`, specify the GroupVersionKind
    * @returns
    */
-  When<T extends GenericClass>(model: T, kind?: GroupVersionKind): WhenSelector<T> {
+  When = <T extends GenericClass>(model: T, kind?: GroupVersionKind): WhenSelector<T> => {
     const matchedKind = modelToGroupVersionKind(model.name);
 
     // If the kind is not specified and the model is not a KubernetesObject, throw an error
@@ -75,6 +115,7 @@ export class Capability implements CapabilityExport {
     }
 
     const binding: Binding = {
+      model,
       // If the kind is not specified, use the matched kind from the model
       kind: kind || matchedKind,
       event: Event.Any,
@@ -88,7 +129,7 @@ export class Capability implements CapabilityExport {
 
     const bindings = this.#bindings;
     const prefix = `${this.#name}: ${model.name}`;
-    const commonChain = { WithLabel, WithAnnotation, Mutate, Validate };
+    const commonChain = { WithLabel, WithAnnotation, Mutate, Validate, Watch };
     const isNotEmpty = (value: object) => Object.keys(value).length > 0;
     const log = (message: string, cbString: string) => {
       const filteredObj = pickBy(isNotEmpty, binding.filters);
@@ -98,8 +139,8 @@ export class Capability implements CapabilityExport {
       Log.debug(cbString, prefix);
     };
 
-    function Validate(validateCallback: ValidateAction<T>): void {
-      if (!isWatchMode) {
+    function Validate(validateCallback: ValidateAction<T>): ValidateActionChain<T> {
+      if (registerAdmission) {
         log("Validate Action", validateCallback.toString());
 
         // Push the binding to the list of bindings for this capability as a new BindingAction
@@ -110,10 +151,12 @@ export class Capability implements CapabilityExport {
           validateCallback,
         });
       }
+
+      return { Watch };
     }
 
     function Mutate(mutateCallback: MutateAction<T>): MutateActionChain<T> {
-      if (!isWatchMode) {
+      if (registerAdmission) {
         log("Mutate Action", mutateCallback.toString());
 
         // Push the binding to the list of bindings for this capability as a new BindingAction
@@ -126,7 +169,19 @@ export class Capability implements CapabilityExport {
       }
 
       // Now only allow adding actions to the same binding
-      return { Validate };
+      return { Watch, Validate };
+    }
+
+    function Watch(watchCallback: WatchAction<T>) {
+      if (registerWatch) {
+        log("Watch Action", watchCallback.toString());
+
+        bindings.push({
+          ...binding,
+          isWatch: true,
+          watchCallback,
+        });
+      }
     }
 
     function InNamespace(...namespaces: string[]): BindingWithName<T> {
@@ -168,5 +223,5 @@ export class Capability implements CapabilityExport {
       IsUpdated: () => bindEvent(Event.Update),
       IsDeleted: () => bindEvent(Event.Delete),
     };
-  }
+  };
 }

package/src/lib/{controller.ts → controller/index.ts}

@@ -5,14 +5,14 @@ import express, { NextFunction } from "express";
 import fs from "fs";
 import https from "https";
 
-import { Capability } from "./capability";
-import { isWatchMode } from "./k8s";
-import { MutateResponse, Request, ValidateResponse } from "./k8s/types";
-import Log from "./logger";
-import { MetricsCollector } from "./metrics";
-import { mutateProcessor } from "./mutate-processor";
-import { ModuleConfig } from "./types";
-import { validateProcessor } from "./validate-processor";
+import { Capability } from "../capability";
+import { MutateResponse, AdmissionRequest, ValidateResponse } from "../k8s";
+import Log from "../logger";
+import { MetricsCollector } from "../metrics";
+import { ModuleConfig, isWatchMode } from "../module";
+import { mutateProcessor } from "../mutate-processor";
+import { validateProcessor } from "../validate-processor";
+import { PeprControllerStore } from "./store";
 
 export class Controller {
   // Track whether the server is running
@@ -30,22 +30,25 @@ export class Controller {
   // Initialized with the constructor
   readonly #config: ModuleConfig;
   readonly #capabilities: Capability[];
-  readonly #beforeHook?: (req: Request) => void;
+  readonly #beforeHook?: (req: AdmissionRequest) => void;
   readonly #afterHook?: (res: MutateResponse) => void;
 
   constructor(
     config: ModuleConfig,
     capabilities: Capability[],
-    beforeHook?: (req: Request) => void,
+    beforeHook?: (req: AdmissionRequest) => void,
     afterHook?: (res: MutateResponse) => void,
+    onReady?: () => void,
   ) {
     this.#config = config;
     this.#capabilities = capabilities;
-    this.#beforeHook = beforeHook;
-    this.#afterHook = afterHook;
 
-    // Bind public methods
-    this.startServer = this.startServer.bind(this);
+    // Initialize the Pepr store for each capability
+    new PeprControllerStore(config, capabilities, () => {
+      this.#bindEndpoints();
+      onReady && onReady();
+      Log.info("✅ Controller startup complete");
+    });
 
     // Middleware for logging requests
     this.#app.use(Controller.#logger);
@@ -55,18 +58,17 @@ export class Controller {
 
     if (beforeHook) {
       Log.info(`Using beforeHook: ${beforeHook}`);
+      this.#beforeHook = beforeHook;
     }
 
     if (afterHook) {
       Log.info(`Using afterHook: ${afterHook}`);
+      this.#afterHook = afterHook;
     }
-
-    // Bind endpoints
-    this.#bindEndpoints();
   }
 
   /** Start the webhook server */
-  startServer(port: number) {
+  startServer = (port: number) => {
     if (this.#running) {
       throw new Error("Cannot start Pepr module: Pepr module was not instantiated with deferStart=true");
     }
@@ -78,7 +80,7 @@ export class Controller {
     };
 
     // Get the API token if not in watch mode
-    if (!isWatchMode) {
+    if (!isWatchMode()) {
       // Get the API token from the environment variable or the mounted secret
       this.#token = process.env.PEPR_API_TOKEN || fs.readFileSync("/app/api-token/value").toString().trim();
       Log.info(`Using API token: ${this.#token}`);
@@ -119,7 +121,7 @@ export class Controller {
         process.exit(0);
       });
     });
-  }
+  };
 
   #bindEndpoints = () => {
     // Health check endpoint
@@ -128,7 +130,7 @@ export class Controller {
     // Metrics endpoint
     this.#app.get("/metrics", this.#metrics);
 
-    if (isWatchMode) {
+    if (isWatchMode()) {
       return;
     }
 
@@ -190,11 +192,11 @@ export class Controller {
     // Create the admission request handler
     return async (req: express.Request, res: express.Response) => {
       // Start the metrics timer
-      const startTime = this.#metricsCollector.observeStart();
+      const startTime = MetricsCollector.observeStart();
 
       try {
         // Get the request from the body or create an empty request
-        const request: Request = req.body?.request || ({} as Request);
+        const request: AdmissionRequest = req.body?.request || ({} as AdmissionRequest);
 
         // Run the before hook if it exists
         this.#beforeHook && this.#beforeHook(request || {});

package/src/lib/controller/store.ts

@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
+
+import { Operation } from "fast-json-patch";
+import { K8s } from "kubernetes-fluent-client";
+import { startsWith } from "ramda";
+
+import { Capability } from "../capability";
+import { PeprStore } from "../k8s";
+import Log from "../logger";
+import { ModuleConfig } from "../module";
+import { DataOp, DataSender, DataStore, Storage } from "../storage";
+
+const namespace = "pepr-system";
+const debounceBackoff = 5000;
+
+export class PeprControllerStore {
+  #name: string;
+  #stores: Record<string, Storage> = {};
+  #sendDebounce: NodeJS.Timeout | undefined;
+  #onReady?: () => void;
+
+  constructor(config: ModuleConfig, capabilities: Capability[], onReady?: () => void) {
+    this.#onReady = onReady;
+
+    // Setup Pepr State bindings
+    this.#name = `pepr-${config.uuid}-store`;
+
+    // Establish the store for each capability
+    for (const { name, registerStore } of capabilities) {
+      // Register the store with the capability
+      const { store } = registerStore();
+
+      // Bind the store sender to the capability
+      store.registerSender(this.#send(name));
+
+      // Store the storage instance
+      this.#stores[name] = store;
+    }
+
+    // Add a jitter to the Store creation to avoid collisions
+    setTimeout(
+      () =>
+        K8s(PeprStore)
+          .InNamespace(namespace)
+          .Get(this.#name)
+          // If the get succeeds, setup the watch
+          .then(this.#setupWatch)
+          // Otherwise, create the resource
+          .catch(this.#createStoreResource),
+      Math.random() * 3000,
+    );
+  }
+
+  #setupWatch = () => {
+    void K8s(PeprStore, { name: this.#name, namespace }).Watch(this.#receive);
+  };
+
+  #receive = (store: PeprStore) => {
+    Log.debug(store, "Pepr Store update");
+
+    // Wrap the update in a debounced function
+    const debounced = () => {
+      // Base64 decode the data
+      const data: DataStore = store.data || {};
+
+      // Loop over each stored capability
+      for (const name of Object.keys(this.#stores)) {
+        // Get the prefix offset for the keys
+        const offset = `${name}-`.length;
+
+        // Get any keys that match the capability name prefix
+        const filtered: DataStore = {};
+
+        // Loop over each key in the secret
+        for (const key of Object.keys(data)) {
+          // Match on the capability name as a prefix
+          if (startsWith(name, key)) {
+            // Strip the prefix and store the value
+            filtered[key.slice(offset)] = data[key];
+          }
+        }
+
+        // Send the data to the receiver callback
+        this.#stores[name].receive(filtered);
+      }
+
+      // Call the onReady callback if this is the first time the secret has been read
+      if (this.#onReady) {
+        this.#onReady();
+        this.#onReady = undefined;
+      }
+    };
+
+    // Debounce the update to 1 second to avoid multiple rapid calls
+    clearTimeout(this.#sendDebounce);
+    this.#sendDebounce = setTimeout(debounced, debounceBackoff);
+  };
+
+  #send = (capabilityName: string) => {
+    const sendCache: Record<string, Operation> = {};
+
+    // Load the sendCache with patch operations
+    const fillCache = (op: DataOp, key: string[], val?: string) => {
+      if (op === "add") {
+        const path = `/data/${capabilityName}-${key}`;
+        const value = val || "";
+        const cacheIdx = [op, path, value].join(":");
+
+        // Add the operation to the cache
+        sendCache[cacheIdx] = { op, path, value };
+
+        return;
+      }
+
+      if (op === "remove") {
+        if (key.length < 1) {
+          throw new Error(`Key is required for REMOVE operation`);
+        }
+
+        for (const k of key) {
+          const path = `/data/${capabilityName}-${k}`;
+          const cacheIdx = [op, path].join(":");
+
+          // Add the operation to the cache
+          sendCache[cacheIdx] = { op, path };
+        }
+
+        return;
+      }
+
+      // If we get here, the operation is not supported
+      throw new Error(`Unsupported operation: ${op}`);
+    };
+
+    // Send the cached updates to the cluster
+    const flushCache = async () => {
+      const indexes = Object.keys(sendCache);
+      const payload = Object.values(sendCache);
+
+      // Loop over each key in the cache and delete it to avoid collisions with other sender calls
+      for (const idx of indexes) {
+        delete sendCache[idx];
+      }
+
+      try {
+        // Send the patch to the cluster
+        await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+      } catch (err) {
+        Log.error(err, "Pepr store update failure");
+
+        // On failure to update, re-add the operations to the cache to be retried
+        for (const idx of indexes) {
+          sendCache[idx] = payload[Number(idx)];
+        }
+      }
+    };
+
+    // Create a sender function for the capability to add/remove data from the store
+    const sender: DataSender = async (op: DataOp, key: string[], val?: string) => {
+      fillCache(op, key, val);
+    };
+
+    // Send any cached updates every debounceBackoff milliseconds
+    setInterval(() => {
+      if (Object.keys(sendCache).length > 0) {
+        Log.debug(sendCache, "Sending updates to Pepr store");
+        void flushCache();
+      }
+    }, debounceBackoff);
+
+    return sender;
+  };
+
+  #createStoreResource = async (e: unknown) => {
+    Log.info(`Pepr store not found, creating...`);
+    Log.debug(e);
+
+    try {
+      await K8s(PeprStore).Apply({
+        metadata: {
+          name: this.#name,
+          namespace,
+        },
+        data: {
+          // JSON Patch will die if the data is empty, so we need to add a placeholder
+          __pepr_do_not_delete__: "k-thx-bye",
+        },
+      });
+
+      // Now that the resource exists, setup the watch
+      this.#setupWatch();
+    } catch (err) {
+      Log.error(err, "Failed to create Pepr store");
+    }
+  };
+}

package/src/lib/filter.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 
-import { Operation, Request } from "./k8s/types";
+import { AdmissionRequest, Operation } from "./k8s";
 import logger from "./logger";
 import { Binding, Event } from "./types";
 
@@ -12,7 +12,7 @@ import { Binding, Event } from "./types";
  * @param req the incoming request
  * @returns
  */
-export function shouldSkipRequest(binding: Binding, req: Request, capabilityNamespaces: string[]) {
+export function shouldSkipRequest(binding: Binding, req: AdmissionRequest, capabilityNamespaces: string[]) {
   const { group, kind, version } = binding.kind || {};
   const { namespaces, labels, annotations, name } = binding.filters || {};
   const operation = req.operation.toUpperCase();

package/src/lib/{k8s/types.ts → k8s.ts}

@@ -1,9 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 
-import { KubernetesListObject, KubernetesObject, V1ObjectMeta } from "@kubernetes/client-node";
-
-export { KubernetesListObject, KubernetesObject };
+import { GenericKind, GroupVersionKind, KubernetesObject, RegisterKind } from "kubernetes-fluent-client";
 
 export enum Operation {
   CREATE = "CREATE",
@@ -13,30 +11,21 @@ export enum Operation {
 }
 
 /**
- * GenericKind is a generic Kubernetes object that can be used to represent any Kubernetes object
- * that is not explicitly supported by Pepr. This can be used on its own or as a base class for
- * other types. See the examples in `HelloPepr.ts` for more information.
+ * PeprStore for internal use by Pepr. This is used to store arbitrary data in the cluster.
  */
-export class GenericKind {
-  apiVersion?: string;
-  kind?: string;
-  metadata?: V1ObjectMeta;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [key: string]: any;
+export class PeprStore extends GenericKind {
+  declare data: {
+    [key: string]: string;
+  };
 }
 
-/**
- * GroupVersionKind unambiguously identifies a kind. It doesn't anonymously include GroupVersion
- * to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling
- **/
-export interface GroupVersionKind {
-  /** The K8s resource kind, e..g "Pod". */
-  readonly kind: string;
-  readonly group: string;
-  readonly version?: string;
-  /** Optional, override the plural name for use in Webhook rules generation */
-  readonly plural?: string;
-}
+export const peprStoreGVK = {
+  kind: "PeprStore",
+  version: "v1",
+  group: "pepr.dev",
+};
+
+RegisterKind(PeprStore, peprStoreGVK);
 
 /**
  * GroupVersionResource unambiguously identifies a resource. It doesn't anonymously include GroupVersion
@@ -51,7 +40,7 @@ export interface GroupVersionResource {
 /**
  * A Kubernetes admission request to be processed by a capability.
  */
-export interface Request<T = KubernetesObject> {
+export interface AdmissionRequest<T = KubernetesObject> {
   /** UID is an identifier for the individual request/response. */
   readonly uid: string;
 
@@ -161,7 +150,7 @@ export interface ValidateResponse extends MutateResponse {
   /** Status contains extra details into why an admission request was denied. This field IS NOT consulted in any way if "Allowed" is "true". */
   status?: {
     /** A machine-readable description of why this operation is in the
-       "Failure" status. If this value is empty there is no information available. */
+         "Failure" status. If this value is empty there is no information available. */
     code: number;
 
     /** A human-readable description of the status of this operation. */