pepr 0.36.0 → 0.37.0

package/src/lib/capability.ts

@@ -2,9 +2,7 @@
 // SPDX-FileCopyrightText: 2023-Present The Pepr Authors
 
 import { GenericClass, GroupVersionKind, modelToGroupVersionKind } from "kubernetes-fluent-client";
-import { WatchAction } from "kubernetes-fluent-client/dist/fluent/types";
 import { pickBy } from "ramda";
-
 import Log from "./logger";
 import { isBuildMode, isDevMode, isWatchMode } from "./module";
 import { PeprStore, Storage } from "./storage";
@@ -20,8 +18,12 @@ import {
   MutateActionChain,
   ValidateAction,
   ValidateActionChain,
+  WatchLogAction,
+  FinalizeAction,
+  FinalizeActionChain,
   WhenSelector,
 } from "./types";
+import { addFinalizer } from "./finalizer";
 
 const registerAdmission = isBuildMode() || !isWatchMode();
 const registerWatch = isBuildMode() || isWatchMode() || isDevMode();
@@ -69,6 +71,10 @@ export class Capability implements CapabilityExport {
     }
   };
 
+  public getScheduleStore() {
+    return this.#scheduleStore;
+  }
+
   /**
    * Store is a key-value data store that can be used to persist data that should be shared
    * between requests. Each capability has its own store, and the data is persisted in Kubernetes
@@ -196,6 +202,8 @@ export class Capability implements CapabilityExport {
       filters: {
         name: "",
         namespaces: [],
+        regexNamespaces: [],
+        regexName: "",
         labels: {},
         annotations: {},
         deletionTimestamp: false,
@@ -204,7 +212,7 @@ export class Capability implements CapabilityExport {
 
     const bindings = this.#bindings;
     const prefix = `${this.#name}: ${model.name}`;
-    const commonChain = { WithLabel, WithAnnotation, WithDeletionTimestamp, Mutate, Validate, Watch, Reconcile };
+    const commonChain = { WithLabel, WithAnnotation, WithDeletionTimestamp, Mutate, Validate, Watch, Reconcile, Alias };
     const isNotEmpty = (value: object) => Object.keys(value).length > 0;
     const log = (message: string, cbString: string) => {
       const filteredObj = pickBy(isNotEmpty, binding.filters);
@@ -218,12 +226,18 @@ export class Capability implements CapabilityExport {
       if (registerAdmission) {
         log("Validate Action", validateCallback.toString());
 
+        // Create the child logger
+        const aliasLogger = Log.child({ alias: binding.alias || "no alias provided" });
+
         // Push the binding to the list of bindings for this capability as a new BindingAction
         // with the callback function to preserve
         bindings.push({
           ...binding,
           isValidate: true,
-          validateCallback,
+          validateCallback: async (req, logger = aliasLogger) => {
+            Log.info(`Executing validate action with alias: ${binding.alias || "no alias provided"}`);
+            return await validateCallback(req, logger);
+          },
         });
       }
 
@@ -234,12 +248,18 @@ export class Capability implements CapabilityExport {
       if (registerAdmission) {
         log("Mutate Action", mutateCallback.toString());
 
+        // Create the child logger
+        const aliasLogger = Log.child({ alias: binding.alias || "no alias provided" });
+
         // Push the binding to the list of bindings for this capability as a new BindingAction
         // with the callback function to preserve
         bindings.push({
           ...binding,
           isMutate: true,
-          mutateCallback,
+          mutateCallback: async (req, logger = aliasLogger) => {
+            Log.info(`Executing mutation action with alias: ${binding.alias || "no alias provided"}`);
+            await mutateCallback(req, logger);
+          },
         });
       }
 
@@ -247,35 +267,93 @@ export class Capability implements CapabilityExport {
       return { Watch, Validate, Reconcile };
     }
 
-    function Watch(watchCallback: WatchAction<T>) {
+    function Watch(watchCallback: WatchLogAction<T>): FinalizeActionChain<T> {
       if (registerWatch) {
         log("Watch Action", watchCallback.toString());
 
+        // Create the child logger and cast it to the expected type
+        const aliasLogger = Log.child({ alias: binding.alias || "no alias provided" }) as typeof Log;
+
+        // Push the binding to the list of bindings for this capability as a new BindingAction
+        // with the callback function to preserve
         bindings.push({
           ...binding,
           isWatch: true,
-          watchCallback,
+          watchCallback: async (update, phase, logger = aliasLogger) => {
+            Log.info(`Executing watch action with alias: ${binding.alias || "no alias provided"}`);
+            await watchCallback(update, phase, logger);
+          },
         });
       }
+      return { Finalize };
     }
 
-    function Reconcile(watchCallback: WatchAction<T>) {
+    function Reconcile(reconcileCallback: WatchLogAction<T>): FinalizeActionChain<T> {
       if (registerWatch) {
-        log("Reconcile Action", watchCallback.toString());
+        log("Reconcile Action", reconcileCallback.toString());
 
+        // Create the child logger and cast it to the expected type
+        const aliasLogger = Log.child({ alias: binding.alias || "no alias provided" }) as typeof Log;
+
+        // Push the binding to the list of bindings for this capability as a new BindingAction
+        // with the callback function to preserve
         bindings.push({
           ...binding,
           isWatch: true,
           isQueue: true,
-          watchCallback,
+          watchCallback: async (update, phase, logger = aliasLogger) => {
+            Log.info(`Executing reconcile action with alias: ${binding.alias || "no alias provided"}`);
+            await reconcileCallback(update, phase, logger);
+          },
         });
       }
+      return { Finalize };
+    }
+
+    function Finalize(finalizeCallback: FinalizeAction<T>): void {
+      log("Finalize Action", finalizeCallback.toString());
+
+      // Create the child logger and cast it to the expected type
+      const aliasLogger = Log.child({ alias: binding.alias || "no alias provided" }) as typeof Log;
+
+      // Add binding to inject Pepr finalizer during admission (Mutate)
+      if (registerAdmission) {
+        const mutateBinding = {
+          ...binding,
+          isMutate: true,
+          isFinalize: true,
+          event: Event.Any,
+          mutateCallback: addFinalizer,
+        };
+        bindings.push(mutateBinding);
+      }
+
+      // Add binding to process finalizer callback / remove Pepr finalizer (Watch)
+      if (registerWatch) {
+        const watchBinding = {
+          ...binding,
+          isWatch: true,
+          isFinalize: true,
+          event: Event.Update,
+          finalizeCallback: async (update: InstanceType<T>, logger = aliasLogger) => {
+            Log.info(`Executing finalize action with alias: ${binding.alias || "no alias provided"}`);
+            await finalizeCallback(update, logger);
+          },
+        };
+        bindings.push(watchBinding);
+      }
     }
 
     function InNamespace(...namespaces: string[]): BindingWithName<T> {
       Log.debug(`Add namespaces filter ${namespaces}`, prefix);
       binding.filters.namespaces.push(...namespaces);
-      return { ...commonChain, WithName };
+      return { ...commonChain, WithName, WithNameRegex };
+    }
+
+    function InNamespaceRegex(...namespaces: RegExp[]): BindingWithName<T> {
+      Log.debug(`Add regex namespaces filter ${namespaces}`, prefix);
+      binding.filters.regexNamespaces.push(...namespaces.map(regex => regex.source));
+      return { ...commonChain, WithName, WithNameRegex };
     }
 
     function WithDeletionTimestamp(): BindingFilter<T> {
@@ -284,6 +362,12 @@ export class Capability implements CapabilityExport {
       return commonChain;
     }
 
+    function WithNameRegex(regexName: RegExp): BindingFilter<T> {
+      Log.debug(`Add regex name filter ${regexName}`, prefix);
+      binding.filters.regexName = regexName.source;
+      return commonChain;
+    }
+
     function WithName(name: string): BindingFilter<T> {
       Log.debug(`Add name filter ${name}`, prefix);
       binding.filters.name = name;
@@ -302,13 +386,22 @@ export class Capability implements CapabilityExport {
       return commonChain;
     }
 
+    function Alias(alias: string) {
+      Log.debug(`Adding prefix alias ${alias}`, prefix);
+      binding.alias = alias;
+      return commonChain;
+    }
+
     function bindEvent(event: Event) {
       binding.event = event;
       return {
         ...commonChain,
         InNamespace,
+        InNamespaceRegex,
         WithName,
+        WithNameRegex,
         WithDeletionTimestamp,
+        Alias,
       };
     }
 

package/src/lib/controller/index.ts

@@ -15,6 +15,9 @@ import { validateProcessor } from "../validate-processor";
 import { PeprControllerStore } from "./store";
 import { ResponseItem } from "../types";
 
+if (!process.env.PEPR_NODE_WARNINGS) {
+  process.removeAllListeners("warning");
+}
 export class Controller {
   // Track whether the server is running
   #running = false;
@@ -108,7 +111,7 @@ export class Controller {
     // Handle EADDRINUSE errors
     server.on("error", (e: { code: string }) => {
       if (e.code === "EADDRINUSE") {
-        Log.warn(
+        Log.info(
           `Address in use, retrying in 2 seconds. If this persists, ensure ${port} is not in use, e.g. "lsof -i :${port}"`,
         );
         setTimeout(() => {
@@ -162,7 +165,7 @@ export class Controller {
     const { token } = req.params;
     if (token !== this.#token) {
       const err = `Unauthorized: invalid token '${token.replace(/[^\w]/g, "_")}'`;
-      Log.warn(err);
+      Log.info(err);
       res.status(401).send(err);
       this.#metricsCollector.alert();
       return;
@@ -227,7 +230,7 @@ export class Controller {
         if (admissionKind === "Mutate") {
           response = await mutateProcessor(this.#config, this.#capabilities, request, reqMetadata);
         } else {
-          response = await validateProcessor(this.#capabilities, request, reqMetadata);
+          response = await validateProcessor(this.#config, this.#capabilities, request, reqMetadata);
         }
 
         // Run the after hook if it exists
@@ -304,7 +307,7 @@ export class Controller {
         duration: `${elapsedTime} ms`,
       };
 
-      res.statusCode >= 300 ? Log.warn(message) : Log.info(message);
+      Log.info(message);
     });
 
     next();

package/src/lib/controller/store.test.ts

@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
+
+import { expect, test, describe, afterEach } from "@jest/globals";
+import * as fc from "fast-check";
+import { redactedStore, redactedPatch } from "./store";
+import { AddOperation } from "fast-json-patch";
+
+const redactedValue = "**redacted**";
+const peprStoreFuzz = fc.record({
+  kind: fc.constant("PeprStore"),
+  apiVersion: fc.constant("v1"),
+  metadata: fc.record({
+    name: fc.string(),
+    namespace: fc.string(),
+  }),
+  data: fc.dictionary(
+    fc.string().filter(str => str !== "__proto__"),
+    fc.string().filter(str => str !== "__proto__"),
+  ),
+});
+describe("Fuzzing redactedStore", () => {
+  afterEach(() => {
+    delete process.env.PEPR_STORE_REDACT_VALUES;
+  });
+
+  test("should redact values if PEPR_STORE_REDACT_VALUES is true", () => {
+    fc.assert(
+      fc.property(peprStoreFuzz, store => {
+        process.env.PEPR_STORE_REDACT_VALUES = "true";
+        const result = redactedStore(store);
+
+        Object.values(result.data).forEach(value => {
+          expect(value).toBe(redactedValue);
+        });
+      }),
+    );
+  });
+
+  test("should not redact values if PEPR_STORE_REDACT_VALUES is not true", () => {
+    fc.assert(
+      fc.property(peprStoreFuzz, store => {
+        process.env.PEPR_STORE_REDACT_VALUES = "false";
+        const result = redactedStore(store);
+        expect(result.data).toEqual(store.data);
+      }),
+    );
+  });
+
+  test("should maintain other properties of the store", () => {
+    fc.assert(
+      fc.property(peprStoreFuzz, store => {
+        const redactionEnabled = fc.boolean();
+        process.env.PEPR_STORE_REDACT_VALUES = redactionEnabled ? "true" : "false";
+
+        const result = redactedStore(store);
+
+        expect(result.kind).toBe(store.kind);
+        expect(result.apiVersion).toBe(store.apiVersion);
+        expect(result.metadata).toEqual(store.metadata);
+      }),
+    );
+  });
+});
+
+const addOperationKeys = [
+  "add:/data/hello-pepr-a:secret",
+  "add:/data/hello-pepr-v2-b:secret",
+  "add:/data/hello-pepr-v2-c:secret",
+  "add:/data/hello-pepr-v2-d:secret",
+  "add:/data/hello-pepr-v2-e:secret",
+  "add:/data/hello-pepr-v2-f:secret",
+];
+const addOperationValues: AddOperation<string>[] = [
+  {
+    op: "add",
+    path: "add:/data/hello-pepr-a",
+    value: "secret",
+  },
+  {
+    op: "add",
+    path: "add:/data/hello-pepr-v2-b",
+    value: "secret",
+  },
+  {
+    op: "add",
+    path: "add:/data/hello-pepr-v2-c",
+    value: "secret",
+  },
+  {
+    op: "add",
+    path: "add:/data/hello-pepr-v2-d",
+    value: "secret",
+  },
+  {
+    op: "add",
+    path: "add:/data/hello-pepr-v2-e",
+    value: "secret",
+  },
+  {
+    op: "add",
+    path: "add:/data/hello-pepr-v2-f",
+    value: "secret",
+  },
+];
+
+describe("redactedPatch", () => {
+  afterEach(() => {
+    delete process.env.PEPR_STORE_REDACT_VALUES;
+  });
+
+  test("should redact keys and values if PEPR_STORE_REDACT_VALUES is true", () => {
+    process.env.PEPR_STORE_REDACT_VALUES = "true";
+    addOperationKeys.forEach((key, i) => {
+      const redactedResult = redactedPatch({ [key]: addOperationValues[i] });
+      for (const [k, v] of Object.entries(redactedResult)) {
+        expect(k).toContain(`:${redactedValue}`);
+        expect(v).toEqual(expect.objectContaining({ value: redactedValue }));
+      }
+    });
+  });
+  test("should not redact keys and values if PEPR_STORE_REDACT_VALUES is not true", () => {
+    addOperationKeys.forEach((key, i) => {
+      const redactedResult = redactedPatch({ [key]: addOperationValues[i] });
+      for (const [k, v] of Object.entries(redactedResult)) {
+        expect(k).not.toContain(`:${redactedValue}`);
+        expect(v).not.toEqual(expect.objectContaining({ value: redactedValue }));
+      }
+    });
+  });
+});

package/src/lib/controller/store.ts

@@ -10,6 +10,7 @@ import { PeprStore } from "../k8s";
 import Log from "../logger";
 import { DataOp, DataSender, DataStore, Storage } from "../storage";
 
+const redactedValue = "**redacted**";
 const namespace = "pepr-system";
 export const debounceBackoff = 5000;
 
@@ -75,7 +76,7 @@ export class PeprControllerStore {
   };
 
   #migrateAndSetupWatch = async (store: PeprStore) => {
-    Log.debug(store, "Pepr Store migration");
+    Log.debug(redactedStore(store), "Pepr Store migration");
     const data: DataStore = store.data || {};
     const migrateCache: Record<string, Operation> = {};
 
@@ -91,7 +92,9 @@ export class PeprControllerStore {
 
       try {
         // Send the patch to the cluster
-        await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+        if (payload.length > 0) {
+          await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+        }
       } catch (err) {
         Log.error(err, "Pepr store update failure");
 
@@ -158,7 +161,7 @@ export class PeprControllerStore {
   };
 
   #receive = (store: PeprStore) => {
-    Log.debug(store, "Pepr Store update");
+    Log.debug(redactedStore(store), "Pepr Store update");
 
     // Wrap the update in a debounced function
     const debounced = () => {
@@ -247,7 +250,9 @@ export class PeprControllerStore {
 
       try {
         // Send the patch to the cluster
-        await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+        if (payload.length > 0) {
+          await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+        }
       } catch (err) {
         Log.error(err, "Pepr store update failure");
 
@@ -270,7 +275,7 @@ export class PeprControllerStore {
     // Send any cached updates every debounceBackoff milliseconds
     setInterval(() => {
       if (Object.keys(sendCache).length > 0) {
-        Log.debug(sendCache, "Sending updates to Pepr store");
+        Log.debug(redactedPatch(sendCache), "Sending updates to Pepr store");
         void flushCache();
       }
     }, debounceBackoff);
@@ -301,3 +306,36 @@ export class PeprControllerStore {
     }
   };
 }
+
+export function redactedStore(store: PeprStore): PeprStore {
+  const redacted = process.env.PEPR_STORE_REDACT_VALUES === "true";
+  return {
+    ...store,
+    data: Object.keys(store.data).reduce((acc: Record<string, string>, key: string) => {
+      acc[key] = redacted ? redactedValue : store.data[key];
+      return acc;
+    }, {}),
+  };
+}
+
+export function redactedPatch(patch: Record<string, Operation> = {}): Record<string, Operation> {
+  const redacted = process.env.PEPR_STORE_REDACT_VALUES === "true";
+
+  if (!redacted) {
+    return patch;
+  }
+
+  const redactedCache: Record<string, Operation> = {};
+
+  Object.keys(patch).forEach(key => {
+    const operation = patch[key];
+    const redactedKey = key.includes(":") ? key.substring(0, key.lastIndexOf(":")) + ":**redacted**" : key;
+    const redactedOperation: Operation = {
+      ...operation,
+      ...("value" in operation ? { value: "**redacted**" } : {}),
+    };
+    redactedCache[redactedKey] = redactedOperation;
+  });
+
+  return redactedCache;
+}