pepr 0.42.3 → 0.44.0-nightly.0

package/src/lib/assets/{yaml.ts → yaml/overridesFile.ts}

@@ -1,20 +1,9 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
-
-import {
-  dumpYaml,
-  V1Deployment,
-  V1MutatingWebhookConfiguration,
-  V1ValidatingWebhookConfiguration,
-} from "@kubernetes/client-node";
+import { genEnv } from "../pods";
+import { ModuleConfig } from "../../core/module";
+import { CapabilityExport } from "../../types";
+import { dumpYaml } from "@kubernetes/client-node";
+import { clusterRole } from "../rbac";
 import { promises as fs } from "fs";
-import { apiTokenSecret, service, tlsSecret, watcherService } from "./networking";
-import { getModuleSecret, getNamespace } from "./pods";
-import { clusterRole, clusterRoleBinding, serviceAccount, storeRole, storeRoleBinding } from "./rbac";
-import { genEnv } from "./pods";
-import { ModuleConfig } from "../core/module";
-import { CapabilityExport } from "../types";
-import { TLSOut } from "../tls";
 
 type CommonOverrideValues = {
   apiToken: string;
@@ -27,20 +16,17 @@ type CommonOverrideValues = {
 type ChartOverrides = CommonOverrideValues & {
   image: string;
 };
-
-type ResourceOverrides = CommonOverrideValues & {
-  path: string;
-  tls: TLSOut;
-};
-
 // Helm Chart overrides file (values.yaml) generated from assets
 export async function overridesFile(
   { hash, name, image, config, apiToken, capabilities }: ChartOverrides,
   path: string,
+  imagePullSecrets: string[],
 ): Promise<void> {
   const rbacOverrides = clusterRole(name, capabilities, config.rbacMode, config.rbac).rules;
 
   const overrides = {
+    imagePullSecrets,
+    additionalIgnoredNamespaces: [],
     rbac: rbacOverrides,
     secrets: {
       apiToken: Buffer.from(apiToken).toString("base64"),
@@ -191,101 +177,3 @@ export async function overridesFile(
 
   await fs.writeFile(path, dumpYaml(overrides, { noRefs: true, forceQuotes: true }));
 }
-export function generateZarfYaml(name: string, image: string, config: ModuleConfig, path: string): string {
-  const zarfCfg = {
-    kind: "ZarfPackageConfig",
-    metadata: {
-      name,
-      description: `Pepr Module: ${config.description}`,
-      url: "https://github.com/defenseunicorns/pepr",
-      version: `${config.appVersion || "0.0.1"}`,
-    },
-    components: [
-      {
-        name: "module",
-        required: true,
-        manifests: [
-          {
-            name: "module",
-            namespace: "pepr-system",
-            files: [path],
-          },
-        ],
-        images: [image],
-      },
-    ],
-  };
-
-  return dumpYaml(zarfCfg, { noRefs: true });
-}
-
-export function generateZarfYamlChart(name: string, image: string, config: ModuleConfig, path: string): string {
-  const zarfCfg = {
-    kind: "ZarfPackageConfig",
-    metadata: {
-      name,
-      description: `Pepr Module: ${config.description}`,
-      url: "https://github.com/defenseunicorns/pepr",
-      version: `${config.appVersion || "0.0.1"}`,
-    },
-    components: [
-      {
-        name: "module",
-        required: true,
-        charts: [
-          {
-            name: "module",
-            namespace: "pepr-system",
-            version: `${config.appVersion || "0.0.1"}`,
-            localPath: path,
-          },
-        ],
-        images: [image],
-      },
-    ],
-  };
-
-  return dumpYaml(zarfCfg, { noRefs: true });
-}
-
-type webhooks = { validate: V1ValidatingWebhookConfiguration | null; mutate: V1MutatingWebhookConfiguration | null };
-type deployments = { default: V1Deployment; watch: V1Deployment | null };
-
-export async function generateAllYaml(
-  webhooks: webhooks,
-  deployments: deployments,
-  assets: ResourceOverrides,
-): Promise<string> {
-  const { name, tls, hash, apiToken, path, config } = assets;
-  const code = await fs.readFile(path);
-
-  const resources = [
-    getNamespace(assets.config.customLabels?.namespace),
-    clusterRole(name, assets.capabilities, config.rbacMode, config.rbac),
-    clusterRoleBinding(name),
-    serviceAccount(name),
-    apiTokenSecret(name, apiToken),
-    tlsSecret(name, tls),
-    deployments.default,
-    service(name),
-    watcherService(name),
-    getModuleSecret(name, code, hash),
-    storeRole(name),
-    storeRoleBinding(name),
-  ];
-
-  if (webhooks.mutate) {
-    resources.push(webhooks.mutate);
-  }
-
-  if (webhooks.validate) {
-    resources.push(webhooks.validate);
-  }
-
-  if (deployments.watch) {
-    resources.push(deployments.watch);
-  }
-
-  // Convert the resources to a single YAML string
-  return resources.map(r => dumpYaml(r, { noRefs: true })).join("---\n");
-}

package/src/lib/core/module.ts

@@ -9,39 +9,44 @@ import { CapabilityExport, AdmissionRequest } from "../types";
 import { setupWatch } from "../processors/watch-processor";
 import { Log } from "../../lib";
 import { V1PolicyRule as PolicyRule } from "@kubernetes/client-node";
+import { resolveIgnoreNamespaces } from "../assets/webhooks";
 
 /** Custom Labels Type for package.json */
-export interface CustomLabels {
-  namespace?: Record<string, string>;
-}
-/** Global configuration for the Pepr runtime. */
-export type ModuleConfig = {
+
+export type CustomLabels = { namespace: Record<string, string> } | Record<string, never>;
+
+/** Configuration that MAY be set a Pepr module's package.json. */
+export type ModuleConfigOptions = {
   /** The Pepr version this module uses */
-  peprVersion?: string;
+  peprVersion: string;
   /** The user-defined version of the module */
-  appVersion?: string;
-  /** A unique identifier for this Pepr module. This is automatically generated by Pepr. */
-  uuid: string;
+  appVersion: string;
   /** A description of the Pepr module and what it does. */
-  description?: string;
+  description: string;
   /** The webhookTimeout */
-  webhookTimeout?: number;
+  webhookTimeout: number;
   /** Reject K8s resource AdmissionRequests on error. */
-  onError?: string;
-  /** Configure global exclusions that will never be processed by Pepr. */
-  alwaysIgnore: WebhookIgnore;
+  onError: string;
   /** Define the log level for the in-cluster controllers */
-  logLevel?: string;
+  logLevel: string;
   /** Propagate env variables to in-cluster controllers */
-  env?: Record<string, string>;
-  /** Custom Labels for Kubernetes Objects */
-  customLabels?: CustomLabels;
+  env: Record<string, string>;
   /** Custom RBAC rules */
-  rbac?: PolicyRule[];
+  rbac: PolicyRule[];
   /** The RBAC mode; if "scoped", generates scoped rules, otherwise uses wildcard rules. */
-  rbacMode?: string;
+  rbacMode: string;
+  /** Custom Labels for Kubernetes Objects */
+  customLabels: CustomLabels;
 };
 
+/** Global configuration for the Pepr runtime. */
+export type ModuleConfig = {
+  /** A unique identifier for this Pepr module. This is automatically generated by Pepr. */
+  uuid: string;
+  /** Configure global exclusions that will never be processed by Pepr. */
+  alwaysIgnore: WebhookIgnore;
+} & Partial<ModuleConfigOptions>;
+
 export type PackageJSON = {
   description: string;
   pepr: ModuleConfig;
@@ -113,7 +118,7 @@ export class PeprModule {
       // Wait for the controller to be ready before setting up watches
       if (isWatchMode() || isDevMode()) {
         try {
-          setupWatch(capabilities, pepr?.alwaysIgnore?.namespaces);
+          setupWatch(capabilities, resolveIgnoreNamespaces(pepr?.alwaysIgnore?.namespaces));
         } catch (e) {
           Log.error(e, "Error setting up watch");
           process.exit(1);

package/src/lib/enums.ts

@@ -19,3 +19,9 @@ export enum Event {
   CREATE_OR_UPDATE = "CREATEORUPDATE",
   ANY = "*",
 }
+
+// Supported webhook types for @kubernetes/client-node's V1MutatingWebhookConfiguration
+export enum WebhookType {
+  MUTATE = "mutate",
+  VALIDATE = "validate",
+}

package/src/lib/processors/mutate-processor.ts

@@ -4,7 +4,7 @@
 import jsonPatch from "fast-json-patch";
 import { kind, KubernetesObject } from "kubernetes-fluent-client";
 import { clone } from "ramda";
-
+import { MeasureWebhookTimeout } from "../telemetry/webhookTimeouts";
 import { Capability } from "../core/capability";
 import { shouldSkipRequest } from "../filter/filter";
 import { MutateResponse } from "../k8s";
@@ -14,7 +14,9 @@ import { ModuleConfig } from "../core/module";
 import { PeprMutateRequest } from "../mutate-request";
 import { base64Encode, convertFromBase64Map, convertToBase64Map } from "../utils";
 import { OnError } from "../../cli/init/enums";
-
+import { resolveIgnoreNamespaces } from "../assets/webhooks";
+import { Operation } from "fast-json-patch";
+import { WebhookType } from "../enums";
 export interface Bindable {
   req: AdmissionRequest;
   config: ModuleConfig;
@@ -138,6 +140,8 @@ export async function mutateProcessor(
   req: AdmissionRequest,
   reqMetadata: Record<string, string>,
 ): Promise<MutateResponse> {
+  const webhookTimer = new MeasureWebhookTimeout(WebhookType.MUTATE);
+  webhookTimer.start(config.webhookTimeout);
   let response: MutateResponse = {
     uid: req.uid,
     warnings: [],
@@ -169,7 +173,7 @@ export async function mutateProcessor(
       bind.binding,
       bind.req,
       bind.namespaces,
-      bind.config?.alwaysIgnore?.namespaces,
+      resolveIgnoreNamespaces(bind.config?.alwaysIgnore?.namespaces),
     );
     if (shouldSkip !== "") {
       Log.debug(shouldSkip);
@@ -206,6 +210,14 @@ export async function mutateProcessor(
   // Compare the original request to the modified request to get the patches
   const patches = jsonPatch.compare(req.object, transformed);
 
+  updateResponsePatchAndWarnings(patches, response);
+
+  Log.debug({ ...reqMetadata, patches }, `Patches generated`);
+  webhookTimer.stop();
+  return response;
+}
+
+export function updateResponsePatchAndWarnings(patches: Operation[], response: MutateResponse): void {
   // Only add the patch if there are patches to apply
   if (patches.length > 0) {
     response.patchType = "JSONPatch";
@@ -218,8 +230,4 @@ export async function mutateProcessor(
   if (response.warnings && response.warnings.length < 1) {
     delete response.warnings;
   }
-
-  Log.debug({ ...reqMetadata, patches }, `Patches generated`);
-
-  return response;
 }

package/src/lib/processors/validate-processor.ts

@@ -10,6 +10,9 @@ import Log from "../telemetry/logger";
 import { convertFromBase64Map } from "../utils";
 import { PeprValidateRequest } from "../validate-request";
 import { ModuleConfig } from "../core/module";
+import { resolveIgnoreNamespaces } from "../assets/webhooks";
+import { MeasureWebhookTimeout } from "../telemetry/webhookTimeouts";
+import { WebhookType } from "../enums";
 
 export async function processRequest(
   binding: Binding,
@@ -57,12 +60,13 @@ export async function validateProcessor(
   req: AdmissionRequest,
   reqMetadata: Record<string, string>,
 ): Promise<ValidateResponse[]> {
+  const webhookTimer = new MeasureWebhookTimeout(WebhookType.VALIDATE);
+  webhookTimer.start(config.webhookTimeout);
   const wrapped = new PeprValidateRequest(req);
   const response: ValidateResponse[] = [];
 
   // If the resource is a secret, decode the data
-  const isSecret = req.kind.version === "v1" && req.kind.kind === "Secret";
-  if (isSecret) {
+  if (req.kind.version === "v1" && req.kind.kind === "Secret") {
     convertFromBase64Map(wrapped.Raw as unknown as kind.Secret);
   }
 
@@ -78,7 +82,12 @@ export async function validateProcessor(
       }
 
       // Continue to the next action without doing anything if this one should be skipped
-      const shouldSkip = shouldSkipRequest(binding, req, namespaces, config?.alwaysIgnore?.namespaces);
+      const shouldSkip = shouldSkipRequest(
+        binding,
+        req,
+        namespaces,
+        resolveIgnoreNamespaces(config?.alwaysIgnore?.namespaces),
+      );
       if (shouldSkip !== "") {
         Log.debug(shouldSkip);
         continue;
@@ -88,6 +97,6 @@ export async function validateProcessor(
       response.push(resp);
     }
   }
-
+  webhookTimer.stop();
   return response;
 }

package/src/lib/telemetry/timeUtils.ts

@@ -0,0 +1 @@
+export const getNow = (): number => performance.now();

package/src/lib/telemetry/webhookTimeouts.ts

@@ -0,0 +1,34 @@
+import { metricsCollector } from "./metrics";
+import { getNow } from "./timeUtils";
+import Log from "./logger";
+import { WebhookType } from "../enums";
+export class MeasureWebhookTimeout {
+  #startTime: number | null = null;
+  #webhookType: string;
+  timeout: number = 0;
+
+  constructor(webhookType: WebhookType) {
+    this.#webhookType = webhookType;
+    metricsCollector.addCounter(`${webhookType}_timeouts`, `Number of ${webhookType} webhook timeouts`);
+  }
+
+  start(timeout: number = 10): void {
+    this.#startTime = getNow();
+    this.timeout = timeout;
+    Log.info(`Starting timer at ${this.#startTime}`);
+  }
+
+  stop(): void {
+    if (this.#startTime === null) {
+      throw new Error("Timer was not started before calling stop.");
+    }
+
+    const elapsedTime = getNow() - this.#startTime;
+    Log.info(`Webhook ${this.#startTime} took ${elapsedTime}ms`);
+    this.#startTime = null;
+
+    if (elapsedTime > this.timeout) {
+      metricsCollector.incCounter(`${this.#webhookType}_timeouts`);
+    }
+  }
+}

package/dist/lib/assets/yaml.d.ts

@@ -1,32 +0,0 @@
-import { V1Deployment, V1MutatingWebhookConfiguration, V1ValidatingWebhookConfiguration } from "@kubernetes/client-node";
-import { ModuleConfig } from "../core/module";
-import { CapabilityExport } from "../types";
-import { TLSOut } from "../tls";
-type CommonOverrideValues = {
-    apiToken: string;
-    capabilities: CapabilityExport[];
-    config: ModuleConfig;
-    hash: string;
-    name: string;
-};
-type ChartOverrides = CommonOverrideValues & {
-    image: string;
-};
-type ResourceOverrides = CommonOverrideValues & {
-    path: string;
-    tls: TLSOut;
-};
-export declare function overridesFile({ hash, name, image, config, apiToken, capabilities }: ChartOverrides, path: string): Promise<void>;
-export declare function generateZarfYaml(name: string, image: string, config: ModuleConfig, path: string): string;
-export declare function generateZarfYamlChart(name: string, image: string, config: ModuleConfig, path: string): string;
-type webhooks = {
-    validate: V1ValidatingWebhookConfiguration | null;
-    mutate: V1MutatingWebhookConfiguration | null;
-};
-type deployments = {
-    default: V1Deployment;
-    watch: V1Deployment | null;
-};
-export declare function generateAllYaml(webhooks: webhooks, deployments: deployments, assets: ResourceOverrides): Promise<string>;
-export {};
-//# sourceMappingURL=yaml.d.ts.map

package/dist/lib/assets/yaml.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"yaml.d.ts","sourceRoot":"","sources":["../../../src/lib/assets/yaml.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,YAAY,EACZ,8BAA8B,EAC9B,gCAAgC,EACjC,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,KAAK,oBAAoB,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,KAAK,cAAc,GAAG,oBAAoB,GAAG;IAC3C,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,iBAAiB,GAAG,oBAAoB,GAAG;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAGF,wBAAsB,aAAa,CACjC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,cAAc,EACrE,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC,CAyJf;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CA0BxG;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CA2B7G;AAED,KAAK,QAAQ,GAAG;IAAE,QAAQ,EAAE,gCAAgC,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,8BAA8B,GAAG,IAAI,CAAA;CAAE,CAAC;AACrH,KAAK,WAAW,GAAG;IAAE,OAAO,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAAA;CAAE,CAAC;AAEzE,wBAAsB,eAAe,CACnC,QAAQ,EAAE,QAAQ,EAClB,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAAC,MAAM,CAAC,CAiCjB"}