pepr 0.32.7 → 0.33.0

package/package.json

@@ -9,7 +9,7 @@
   "engines": {
     "node": ">=18.0.0"
   },
-  "version": "0.32.7",
+  "version": "0.33.0",
   "main": "dist/lib.js",
   "types": "dist/lib.d.ts",
   "scripts": {
@@ -35,8 +35,9 @@
     "@types/ramda": "0.30.1",
     "express": "4.19.2",
     "fast-json-patch": "3.1.1",
-    "kubernetes-fluent-client": "2.6.4",
-    "pino": "9.2.0",
+    "json-pointer": "^0.6.2",
+    "kubernetes-fluent-client": "2.6.5",
+    "pino": "9.3.1",
     "pino-pretty": "11.2.1",
     "prom-client": "15.1.3",
     "ramda": "0.30.1"
@@ -46,16 +47,18 @@
     "@commitlint/config-conventional": "19.2.2",
     "@fast-check/jest": "^1.8.2",
     "@jest/globals": "29.7.0",
-    "@types/eslint": "8.56.10",
+    "@types/eslint": "9.6.0",
     "@types/express": "4.17.21",
+    "@types/json-pointer": "^1.0.34",
     "@types/node": "18.x.x",
     "@types/node-forge": "1.3.11",
     "@types/prompts": "2.4.9",
-    "fast-check": "^3.19.0",
     "@types/uuid": "10.0.0",
+    "fast-check": "^3.19.0",
     "jest": "29.7.0",
+    "js-yaml": "^4.1.0",
     "nock": "13.5.4",
-    "ts-jest": "29.2.2"
+    "ts-jest": "29.2.3"
   },
   "peerDependencies": {
     "@typescript-eslint/eslint-plugin": "6.15.0",

package/src/lib/assets/yaml.ts

@@ -4,15 +4,26 @@
 import { dumpYaml } from "@kubernetes/client-node";
 import crypto from "crypto";
 import { promises as fs } from "fs";
-
 import { Assets } from ".";
 import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking";
 import { deployment, moduleSecret, namespace, watcher } from "./pods";
 import { clusterRole, clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac";
 import { webhookConfig } from "./webhooks";
+import { mergePkgJSONEnv, envMapToArray } from "../helpers";
 
 // Helm Chart overrides file (values.yaml) generated from assets
 export async function overridesFile({ hash, name, image, config, apiToken }: Assets, path: string) {
+  const pkgJSONAdmissionEnv = {
+    PEPR_WATCH_MODE: "false",
+    PEPR_PRETTY_LOG: "false",
+    LOG_LEVEL: "info",
+  };
+  const pkgJSONWatchEnv = {
+    PEPR_WATCH_MODE: "true",
+    PEPR_PRETTY_LOG: "false",
+    LOG_LEVEL: "info",
+  };
+
   const overrides = {
     secrets: {
       apiToken: Buffer.from(apiToken).toString("base64"),
@@ -29,11 +40,7 @@ export async function overridesFile({ hash, name, image, config, apiToken }: Ass
       terminationGracePeriodSeconds: 5,
       failurePolicy: config.onError === "reject" ? "Fail" : "Ignore",
       webhookTimeout: config.webhookTimeout,
-      env: [
-        { name: "PEPR_WATCH_MODE", value: "false" },
-        { name: "PEPR_PRETTY_LOG", value: "false" },
-        { name: "LOG_LEVEL", value: "info" },
-      ],
+      env: envMapToArray(mergePkgJSONEnv(pkgJSONAdmissionEnv, config.env)),
       image,
       annotations: {
         "pepr.dev/description": `${config.description}` || "",
@@ -77,11 +84,7 @@ export async function overridesFile({ hash, name, image, config, apiToken }: Ass
     },
     watcher: {
       terminationGracePeriodSeconds: 5,
-      env: [
-        { name: "PEPR_WATCH_MODE", value: "true" },
-        { name: "PEPR_PRETTY_LOG", value: "false" },
-        { name: "LOG_LEVEL", value: "info" },
-      ],
+      env: envMapToArray(mergePkgJSONEnv(pkgJSONWatchEnv, config.env)),
       image,
       annotations: {
         "pepr.dev/description": `${config.description}` || "",
@@ -124,12 +127,6 @@ export async function overridesFile({ hash, name, image, config, apiToken }: Ass
       podAnnotations: {},
     },
   };
-  if (process.env.PEPR_MODE === "dev") {
-    overrides.admission.env.push({ name: "ZARF_VAR", value: "###ZARF_VAR_THING###" });
-    overrides.watcher.env.push({ name: "ZARF_VAR", value: "###ZARF_VAR_THING###" });
-    overrides.admission.env.push({ name: "MY_CUSTOM_VAR", value: "example-value" });
-    overrides.watcher.env.push({ name: "MY_CUSTOM_VAR", value: "example-value" });
-  }
 
   await fs.writeFile(path, dumpYaml(overrides, { noRefs: true, forceQuotes: true }));
 }

package/src/lib/controller/store.ts

@@ -61,8 +61,8 @@ export class PeprControllerStore {
         K8s(PeprStore)
           .InNamespace(namespace)
           .Get(this.#name)
-          // If the get succeeds, setup the watch
-          .then(this.#setupWatch)
+          // If the get succeeds, migrate and setup the watch
+          .then(async (store: PeprStore) => await this.#migrateAndSetupWatch(store))
           // Otherwise, create the resource
           .catch(this.#createStoreResource),
       Math.random() * 3000,
@@ -74,6 +74,91 @@ export class PeprControllerStore {
     watcher.start().catch(e => Log.error(e, "Error starting Pepr store watch"));
   };
 
+  #migrateAndSetupWatch = async (store: PeprStore) => {
+    Log.debug(store, "Pepr Store migration");
+    const data: DataStore = store.data || {};
+    const migrateCache: Record<string, Operation> = {};
+
+    // Send the cached updates to the cluster
+    const flushCache = async () => {
+      const indexes = Object.keys(migrateCache);
+      const payload = Object.values(migrateCache);
+
+      // Loop over each key in the cache and delete it to avoid collisions with other sender calls
+      for (const idx of indexes) {
+        delete migrateCache[idx];
+      }
+
+      try {
+        // Send the patch to the cluster
+        await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+      } catch (err) {
+        Log.error(err, "Pepr store update failure");
+
+        if (err.status === 422) {
+          Object.keys(migrateCache).forEach(key => delete migrateCache[key]);
+        } else {
+          // On failure to update, re-add the operations to the cache to be retried
+          for (const idx of indexes) {
+            migrateCache[idx] = payload[Number(idx)];
+          }
+        }
+      }
+    };
+
+    const fillCache = (name: string, op: DataOp, key: string[], val?: string) => {
+      if (op === "add") {
+        // adjust the path for the capability
+        const path = `/data/${name}-v2-${key}`;
+        const value = val || "";
+        const cacheIdx = [op, path, value].join(":");
+
+        // Add the operation to the cache
+        migrateCache[cacheIdx] = { op, path, value };
+
+        return;
+      }
+
+      if (op === "remove") {
+        if (key.length < 1) {
+          throw new Error(`Key is required for REMOVE operation`);
+        }
+
+        for (const k of key) {
+          const path = `/data/${name}-${k}`;
+          const cacheIdx = [op, path].join(":");
+
+          // Add the operation to the cache
+          migrateCache[cacheIdx] = { op, path };
+        }
+
+        return;
+      }
+
+      // If we get here, the operation is not supported
+      throw new Error(`Unsupported operation: ${op}`);
+    };
+
+    for (const name of Object.keys(this.#stores)) {
+      // Get the prefix offset for the keys
+      const offset = `${name}-`.length;
+
+      // Loop over each key in the store
+      for (const key of Object.keys(data)) {
+        // Match on the capability name as a prefix for non v2 keys
+        if (startsWith(name, key) && !startsWith(`${name}-v2`, key)) {
+          // populate migrate cache
+          fillCache(name, "remove", [key.slice(offset)], data[key]);
+          fillCache(name, "add", [key.slice(offset)], data[key]);
+        }
+      }
+
+      // await K8s(PeprStore, { namespace, name: this.#name }).Patch(payload);
+    }
+    await flushCache();
+    this.#setupWatch();
+  };
+
   #receive = (store: PeprStore) => {
     Log.debug(store, "Pepr Store update");
 
@@ -121,6 +206,7 @@ export class PeprControllerStore {
     // Load the sendCache with patch operations
     const fillCache = (op: DataOp, key: string[], val?: string) => {
       if (op === "add") {
+        // adjust the path for the capability
         const path = `/data/${capabilityName}-${key}`;
         const value = val || "";
         const cacheIdx = [op, path, value].join(":");

package/src/lib/helpers.ts

@@ -6,7 +6,24 @@ import { K8s, KubernetesObject, kind } from "kubernetes-fluent-client";
 import Log from "./logger";
 import { Binding, CapabilityExport } from "./types";
 import { sanitizeResourceName } from "../sdk/sdk";
+import { mergeDeepRight } from "ramda";
 
+export function mergePkgJSONEnv(env: Record<string, string>, pkgJSONEnv?: Record<string, string>) {
+  if (!pkgJSONEnv) {
+    return env;
+  }
+  // Cannot override watch mode because it is critical to deployments
+  Object.keys(pkgJSONEnv).forEach(key => {
+    if (key === "PEPR_WATCH_MODE") {
+      delete pkgJSONEnv[key];
+    }
+  });
+  return mergeDeepRight(env, pkgJSONEnv);
+}
+
+export function envMapToArray(env: Record<string, string>) {
+  return Object.entries(env).map(([key, value]) => ({ name: key, value }));
+}
 export class ValidationError extends Error {}
 
 export function validateCapabilityNames(capabilities: CapabilityExport[] | undefined): void {

package/src/lib/storage.ts

@@ -3,7 +3,7 @@
 
 import { clone } from "ramda";
 import Log from "./logger";
-
+import pointer from "json-pointer";
 export type DataOp = "add" | "remove";
 export type DataStore = Record<string, string>;
 export type DataSender = (op: DataOp, keys: string[], value?: string) => void;
@@ -11,6 +11,15 @@ export type DataReceiver = (data: DataStore) => void;
 export type Unsubscribe = () => void;
 
 const MAX_WAIT_TIME = 15000;
+const STORE_VERSION_PREFIX = "v2";
+
+export function v2StoreKey(key: string) {
+  return `${STORE_VERSION_PREFIX}-${pointer.escape(key)}`;
+}
+
+export function stripV2Prefix(key: string) {
+  return key.replace(/^v2-/, "");
+}
 export interface PeprStore {
   /**
    * Returns the current value associated with the given key, or null if the given key does not exist.
@@ -60,6 +69,7 @@ export interface PeprStore {
  *
  * The API is similar to the [Storage API](https://developer.mozilla.org/docs/Web/API/Storage)
  */
+
 export class Storage implements PeprStore {
   #store: DataStore = {};
   #send!: DataSender;
@@ -85,8 +95,11 @@ export class Storage implements PeprStore {
   };
 
   getItem = (key: string) => {
-    // Return null if the value is the empty string
-    return this.#store[key] || null;
+    const result = this.#store[v2StoreKey(key)] || null;
+    if (result !== null && typeof result !== "function" && typeof result !== "object") {
+      return result;
+    }
+    return null;
   };
 
   clear = () => {
@@ -94,11 +107,11 @@ export class Storage implements PeprStore {
   };
 
   removeItem = (key: string) => {
-    this.#dispatchUpdate("remove", [key]);
+    this.#dispatchUpdate("remove", [v2StoreKey(key)]);
   };
 
   setItem = (key: string, value: string) => {
-    this.#dispatchUpdate("add", [key], value);
+    this.#dispatchUpdate("add", [v2StoreKey(key)], value);
   };
 
   /**
@@ -110,10 +123,10 @@ export class Storage implements PeprStore {
    * @returns
    */
   setItemAndWait = (key: string, value: string) => {
-    this.#dispatchUpdate("add", [key], value);
+    this.#dispatchUpdate("add", [v2StoreKey(key)], value);
     return new Promise<void>((resolve, reject) => {
       const unsubscribe = this.subscribe(data => {
-        if (data[key] === value) {
+        if (data[`${v2StoreKey(key)}`] === value) {
           unsubscribe();
           resolve();
         }
@@ -135,10 +148,10 @@ export class Storage implements PeprStore {
    * @returns
    */
   removeItemAndWait = (key: string) => {
-    this.#dispatchUpdate("remove", [key]);
+    this.#dispatchUpdate("remove", [v2StoreKey(key)]);
     return new Promise<void>((resolve, reject) => {
       const unsubscribe = this.subscribe(data => {
-        if (!Object.hasOwn(data, key)) {
+        if (!Object.hasOwn(data, `${v2StoreKey(key)}`)) {
           unsubscribe();
           resolve();
         }