pepr 0.32.7 → 0.34.0

package/dist/lib/validate-request.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate-request.test.d.ts","sourceRoot":"","sources":["../../src/lib/validate-request.test.ts"],"names":[],"mappings":""}

package/dist/lib/watch-processor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"watch-processor.d.ts","sourceRoot":"","sources":["../../src/lib/watch-processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAO,gBAAgB,EAAY,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEvF,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAwB1C;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,YAAY,EAAE,UAAU,EAAE,QAMpD;AA6ED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,GAAE,MAAW,EAAE,GAAG,CAAC,EAAE,gBAAgB,QAOtF"}
+{"version":3,"file":"watch-processor.d.ts","sourceRoot":"","sources":["../../src/lib/watch-processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAO,gBAAgB,EAAY,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEvF,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AA4B1C;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,YAAY,EAAE,UAAU,EAAE,QAMpD;AAyFD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,GAAE,MAAW,EAAE,GAAG,CAAC,EAAE,gBAAgB,QAOtF"}

package/dist/lib/watch-processor.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=watch-processor.test.d.ts.map

package/dist/lib/watch-processor.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"watch-processor.test.d.ts","sourceRoot":"","sources":["../../src/lib/watch-processor.test.ts"],"names":[],"mappings":""}

package/dist/lib.js

@@ -88,13 +88,17 @@ var loggingPrefix = "MetricsCollector";
 var MetricsCollector = class {
   #registry;
   #counters = /* @__PURE__ */ new Map();
+  #gauges = /* @__PURE__ */ new Map();
   #summaries = /* @__PURE__ */ new Map();
   #prefix;
+  #cacheMissWindows = /* @__PURE__ */ new Map();
   #metricNames = {
     errors: "errors",
     alerts: "alerts",
-    mutate: "Mutate",
-    validate: "Validate"
+    mutate: "mutate",
+    validate: "validate",
+    cacheMiss: "cache_miss",
+    resyncFailureCount: "resync_failure_count"
   };
   /**
    * Creates a MetricsCollector instance with prefixed metrics.
@@ -107,9 +111,11 @@ var MetricsCollector = class {
     this.addCounter(this.#metricNames.alerts, "Mutation/Validate bad api token received");
     this.addSummary(this.#metricNames.mutate, "Mutation operation summary");
     this.addSummary(this.#metricNames.validate, "Validation operation summary");
+    this.addGauge(this.#metricNames.cacheMiss, "Number of cache misses per window", ["window"]);
+    this.addGauge(this.#metricNames.resyncFailureCount, "Number of failures per resync operation", ["count"]);
   }
   #getMetricName = (name) => `${this.#prefix}_${name}`;
-  #addMetric = (collection, MetricType, name, help) => {
+  #addMetric = (collection, MetricType, name, help, labelNames) => {
     if (collection.has(this.#getMetricName(name))) {
       logger_default.debug(`Metric for ${name} already exists`, loggingPrefix);
       return;
@@ -117,19 +123,26 @@ var MetricsCollector = class {
     const metric = new MetricType({
       name: this.#getMetricName(name),
       help,
-      registers: [this.#registry]
+      registers: [this.#registry],
+      labelNames
     });
     collection.set(this.#getMetricName(name), metric);
   };
   addCounter = (name, help) => {
-    this.#addMetric(this.#counters, import_prom_client.default.Counter, name, help);
+    this.#addMetric(this.#counters, import_prom_client.default.Counter, name, help, []);
   };
   addSummary = (name, help) => {
-    this.#addMetric(this.#summaries, import_prom_client.default.Summary, name, help);
+    this.#addMetric(this.#summaries, import_prom_client.default.Summary, name, help, []);
+  };
+  addGauge = (name, help, labelNames) => {
+    this.#addMetric(this.#gauges, import_prom_client.default.Gauge, name, help, labelNames);
   };
   incCounter = (name) => {
     this.#counters.get(this.#getMetricName(name))?.inc();
   };
+  incGauge = (name, labels, value = 1) => {
+    this.#gauges.get(this.#getMetricName(name))?.inc(labels || {}, value);
+  };
   /**
    * Increments the error counter.
    */
@@ -158,7 +171,42 @@ var MetricsCollector = class {
   static observeStart() {
     return import_perf_hooks.performance.now();
   }
+  /**
+   * Increments the cache miss gauge for a given label.
+   * @param label - The label for the cache miss.
+   */
+  incCacheMiss = (window) => {
+    this.incGauge(this.#metricNames.cacheMiss, { window });
+  };
+  /**
+   * Increments the retry count gauge.
+   * @param count - The count to increment by.
+   */
+  incRetryCount = (count) => {
+    this.incGauge(this.#metricNames.resyncFailureCount, { count });
+  };
+  /**
+   * Initializes the cache miss gauge for a given label.
+   * @param label - The label for the cache miss.
+   */
+  initCacheMissWindow = (window) => {
+    this.#rollCacheMissWindows();
+    this.#gauges.get(this.#getMetricName(this.#metricNames.cacheMiss))?.set({ window }, 0);
+    this.#cacheMissWindows.set(window, 0);
+  };
+  /**
+   * Manages the size of the cache miss gauge map.
+   */
+  #rollCacheMissWindows = () => {
+    const maxCacheMissWindows = process.env.PEPR_MAX_CACHE_MISS_WINDOWS ? parseInt(process.env.PEPR_MAX_CACHE_MISS_WINDOWS, 10) : void 0;
+    if (maxCacheMissWindows !== void 0 && this.#cacheMissWindows.size >= maxCacheMissWindows) {
+      const firstKey = this.#cacheMissWindows.keys().next().value;
+      this.#cacheMissWindows.delete(firstKey);
+      this.#gauges.get(this.#getMetricName(this.#metricNames.cacheMiss))?.remove({ window: firstKey });
+    }
+  };
 };
+var metricsCollector = new MetricsCollector("pepr");
 
 // src/lib/mutate-processor.ts
 var import_fast_json_patch = __toESM(require("fast-json-patch"));
@@ -665,7 +713,7 @@ var PeprControllerStore = class {
       }
     }
     setTimeout(
-      () => (0, import_kubernetes_fluent_client2.K8s)(PeprStore).InNamespace(namespace).Get(this.#name).then(this.#setupWatch).catch(this.#createStoreResource),
+      () => (0, import_kubernetes_fluent_client2.K8s)(PeprStore).InNamespace(namespace).Get(this.#name).then(async (store) => await this.#migrateAndSetupWatch(store)).catch(this.#createStoreResource),
       Math.random() * 3e3
     );
   }
@@ -673,6 +721,62 @@ var PeprControllerStore = class {
     const watcher = (0, import_kubernetes_fluent_client2.K8s)(PeprStore, { name: this.#name, namespace }).Watch(this.#receive);
     watcher.start().catch((e) => logger_default.error(e, "Error starting Pepr store watch"));
   };
+  #migrateAndSetupWatch = async (store) => {
+    logger_default.debug(store, "Pepr Store migration");
+    const data = store.data || {};
+    const migrateCache = {};
+    const flushCache = async () => {
+      const indexes = Object.keys(migrateCache);
+      const payload = Object.values(migrateCache);
+      for (const idx of indexes) {
+        delete migrateCache[idx];
+      }
+      try {
+        await (0, import_kubernetes_fluent_client2.K8s)(PeprStore, { namespace, name: this.#name }).Patch(payload);
+      } catch (err) {
+        logger_default.error(err, "Pepr store update failure");
+        if (err.status === 422) {
+          Object.keys(migrateCache).forEach((key) => delete migrateCache[key]);
+        } else {
+          for (const idx of indexes) {
+            migrateCache[idx] = payload[Number(idx)];
+          }
+        }
+      }
+    };
+    const fillCache = (name, op, key, val) => {
+      if (op === "add") {
+        const path = `/data/${name}-v2-${key}`;
+        const value = val || "";
+        const cacheIdx = [op, path, value].join(":");
+        migrateCache[cacheIdx] = { op, path, value };
+        return;
+      }
+      if (op === "remove") {
+        if (key.length < 1) {
+          throw new Error(`Key is required for REMOVE operation`);
+        }
+        for (const k of key) {
+          const path = `/data/${name}-${k}`;
+          const cacheIdx = [op, path].join(":");
+          migrateCache[cacheIdx] = { op, path };
+        }
+        return;
+      }
+      throw new Error(`Unsupported operation: ${op}`);
+    };
+    for (const name of Object.keys(this.#stores)) {
+      const offset = `${name}-`.length;
+      for (const key of Object.keys(data)) {
+        if ((0, import_ramda3.startsWith)(name, key) && !(0, import_ramda3.startsWith)(`${name}-v2`, key)) {
+          fillCache(name, "remove", [key.slice(offset)], data[key]);
+          fillCache(name, "add", [key.slice(offset)], data[key]);
+        }
+      }
+    }
+    await flushCache();
+    this.#setupWatch();
+  };
   #receive = (store) => {
     logger_default.debug(store, "Pepr Store update");
     const debounced = () => {
@@ -774,7 +878,7 @@ var Controller = class _Controller {
   // Track whether the server is running
   #running = false;
   // Metrics collector
-  #metricsCollector = new MetricsCollector("pepr");
+  #metricsCollector = metricsCollector;
   // The token used to authenticate requests
   #token = "";
   // The express app instance
@@ -1178,9 +1282,10 @@ var Queue = class {
 
 // src/lib/watch-processor.ts
 var watchCfg = {
-  retryMax: process.env.PEPR_RETRYMAX ? parseInt(process.env.PEPR_RETRYMAX, 10) : 5,
-  retryDelaySec: process.env.PEPR_RETRYDELAYSECONDS ? parseInt(process.env.PEPR_RETRYDELAYSECONDS, 10) : 5,
-  resyncIntervalSec: process.env.PEPR_RESYNCINTERVALSECONDS ? parseInt(process.env.PEPR_RESYNCINTERVALSECONDS, 10) : 300
+  resyncFailureMax: process.env.PEPR_RESYNC_FAILURE_MAX ? parseInt(process.env.PEPR_RESYNC_FAILURE_MAX, 10) : 5,
+  resyncDelaySec: process.env.PEPR_RESYNC_DELAY_SECONDS ? parseInt(process.env.PEPR_RESYNC_DELAY_SECONDS, 10) : 5,
+  lastSeenLimitSeconds: process.env.PEPR_LAST_SEEN_LIMIT_SECONDS ? parseInt(process.env.PEPR_LAST_SEEN_LIMIT_SECONDS, 10) : 300,
+  relistIntervalSec: process.env.PEPR_RELIST_INTERVAL_SECONDS ? parseInt(process.env.PEPR_RELIST_INTERVAL_SECONDS, 10) : 1800
 };
 var eventToPhaseMap = {
   ["CREATE" /* Create */]: [import_types2.WatchPhase.Added],
@@ -1238,6 +1343,15 @@ async function runBinding(binding, capabilityNamespaces) {
   watcher.events.on(import_kubernetes_fluent_client5.WatchEvent.NETWORK_ERROR, (err) => logEvent(import_kubernetes_fluent_client5.WatchEvent.NETWORK_ERROR, err.message));
   watcher.events.on(import_kubernetes_fluent_client5.WatchEvent.LIST_ERROR, (err) => logEvent(import_kubernetes_fluent_client5.WatchEvent.LIST_ERROR, err.message));
   watcher.events.on(import_kubernetes_fluent_client5.WatchEvent.LIST, (list) => logEvent(import_kubernetes_fluent_client5.WatchEvent.LIST, JSON.stringify(list, void 0, 2)));
+  watcher.events.on(import_kubernetes_fluent_client5.WatchEvent.CACHE_MISS, (windowName) => {
+    metricsCollector.incCacheMiss(windowName);
+  });
+  watcher.events.on(import_kubernetes_fluent_client5.WatchEvent.INIT_CACHE_MISS, (windowName) => {
+    metricsCollector.initCacheMissWindow(windowName);
+  });
+  watcher.events.on(import_kubernetes_fluent_client5.WatchEvent.INC_RESYNC_FAILURE_COUNT, (retryCount) => {
+    metricsCollector.incRetryCount(retryCount);
+  });
   try {
     await watcher.start();
   } catch (err) {
@@ -1316,7 +1430,12 @@ var PeprModule = class {
 
 // src/lib/storage.ts
 var import_ramda5 = require("ramda");
+var import_json_pointer = __toESM(require("json-pointer"));
 var MAX_WAIT_TIME = 15e3;
+var STORE_VERSION_PREFIX = "v2";
+function v2StoreKey(key) {
+  return `${STORE_VERSION_PREFIX}-${import_json_pointer.default.escape(key)}`;
+}
 var Storage = class {
   #store = {};
   #send;
@@ -1335,16 +1454,20 @@ var Storage = class {
     }
   };
   getItem = (key) => {
-    return this.#store[key] || null;
+    const result = this.#store[v2StoreKey(key)] || null;
+    if (result !== null && typeof result !== "function" && typeof result !== "object") {
+      return result;
+    }
+    return null;
   };
   clear = () => {
     this.#dispatchUpdate("remove", Object.keys(this.#store));
   };
   removeItem = (key) => {
-    this.#dispatchUpdate("remove", [key]);
+    this.#dispatchUpdate("remove", [v2StoreKey(key)]);
   };
   setItem = (key, value) => {
-    this.#dispatchUpdate("add", [key], value);
+    this.#dispatchUpdate("add", [v2StoreKey(key)], value);
   };
   /**
    * Creates a promise and subscribes to the store, the promise resolves when
@@ -1355,10 +1478,10 @@ var Storage = class {
    * @returns
    */
   setItemAndWait = (key, value) => {
-    this.#dispatchUpdate("add", [key], value);
+    this.#dispatchUpdate("add", [v2StoreKey(key)], value);
     return new Promise((resolve, reject) => {
       const unsubscribe = this.subscribe((data) => {
-        if (data[key] === value) {
+        if (data[`${v2StoreKey(key)}`] === value) {
           unsubscribe();
           resolve();
         }
@@ -1377,10 +1500,10 @@ var Storage = class {
    * @returns
    */
   removeItemAndWait = (key) => {
-    this.#dispatchUpdate("remove", [key]);
+    this.#dispatchUpdate("remove", [v2StoreKey(key)]);
     return new Promise((resolve, reject) => {
       const unsubscribe = this.subscribe((data) => {
-        if (!Object.hasOwn(data, key)) {
+        if (!Object.hasOwn(data, `${v2StoreKey(key)}`)) {
           unsubscribe();
           resolve();
         }
@@ -1485,8 +1608,7 @@ var OnSchedule = class {
   getDuration() {
     switch (this.unit) {
       case "seconds":
-        if (this.every < 10)
-          throw new Error("10 Seconds in the smallest interval allowed");
+        if (this.every < 10) throw new Error("10 Seconds in the smallest interval allowed");
         this.duration = 1e3 * this.every;
         break;
       case "minutes":