pepr 0.33.0 → 0.34.0

package/src/cli/build.ts

@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
+
+import { execSync, execFileSync } from "child_process";
+import { BuildOptions, BuildResult, analyzeMetafile, context } from "esbuild";
+import { promises as fs } from "fs";
+import { basename, dirname, extname, resolve } from "path";
+import { createDockerfile } from "../lib/included-files";
+import { Assets } from "../lib/assets";
+import { dependencies, version } from "./init/templates";
+import { RootCmd } from "./root";
+import { peprFormat } from "./format";
+import { Option } from "commander";
+import { createDirectoryIfNotExists, validateCapabilityNames, parseTimeout } from "../lib/helpers";
+import { sanitizeResourceName } from "../sdk/sdk";
+
+const peprTS = "pepr.ts";
+let outputDir: string = "dist";
+export type Reloader = (opts: BuildResult<BuildOptions>) => void | Promise<void>;
+
+export default function (program: RootCmd) {
+  program
+    .command("build")
+    .description("Build a Pepr Module for deployment")
+    .option("-e, --entry-point [file]", "Specify the entry point file to build with.", peprTS)
+    .option(
+      "-n, --no-embed",
+      "Disables embedding of deployment files into output module.  Useful when creating library modules intended solely for reuse/distribution via NPM.",
+    )
+    .option(
+      "-i, --custom-image <custom-image>",
+      "Custom Image: Use custom image for Admission and Watch Deployments.",
+    )
+    .option(
+      "-r, --registry-info [<registry>/<username>]",
+      "Registry Info: Image registry and username. Note: You must be signed into the registry",
+    )
+    .option("-o, --output-dir <output directory>", "Define where to place build output")
+    .option(
+      "--timeout <timeout>",
+      "How long the API server should wait for a webhook to respond before treating the call as a failure",
+      parseTimeout,
+    )
+    .option(
+      "-v, --version <version>. Example: '0.27.3'",
+      "The version of the Pepr image to use in the deployment manifests.",
+    )
+    .option(
+      "--withPullSecret <imagePullSecret>",
+      "Image Pull Secret: Use image pull secret for controller Deployment.",
+    )
+
+    .addOption(
+      new Option(
+        "--registry <GitHub|Iron Bank>",
+        "Container registry: Choose container registry for deployment manifests. Can't be used with --custom-image.",
+      ).choices(["GitHub", "Iron Bank"]),
+    )
+
+    .addOption(
+      new Option(
+        "-z, --zarf [manifest|chart]",
+        "Zarf package type: manifest, chart (default: manifest)",
+      )
+        .choices(["manifest", "chart"])
+        .default("manifest"),
+    )
+    .addOption(
+      new Option("--rbac-mode [admin|scoped]", "Rbac Mode: admin, scoped (default: admin)")
+        .choices(["admin", "scoped"])
+        .default("admin"),
+    )
+    .action(async opts => {
+      // assign custom output directory if provided
+      if (opts.outputDir) {
+        outputDir = opts.outputDir;
+        createDirectoryIfNotExists(outputDir).catch(error => {
+          console.error(`Error creating output directory: ${error.message}`);
+          process.exit(1);
+        });
+      }
+
+      // Build the module
+      const { cfg, path, uuid } = await buildModule(undefined, opts.entryPoint, opts.embed);
+
+      // Files to include in controller image for WASM support
+      const { includedFiles } = cfg.pepr;
+
+      let image: string = "";
+
+      // Build Kubernetes manifests with custom image
+      if (opts.customImage) {
+        if (opts.registry) {
+          console.error(`Custom Image and registry cannot be used together.`);
+          process.exit(1);
+        }
+        image = opts.customImage;
+      }
+
+      // Check if there is a custom timeout defined
+      if (opts.timeout !== undefined) {
+        cfg.pepr.webhookTimeout = opts.timeout;
+      }
+
+      if (opts.registryInfo !== undefined) {
+        console.info(`Including ${includedFiles.length} files in controller image.`);
+
+        // for journey test to make sure the image is built
+        image = `${opts.registryInfo}/custom-pepr-controller:${cfg.pepr.peprVersion}`;
+
+        // only actually build/push if there are files to include
+        if (includedFiles.length > 0) {
+          await createDockerfile(cfg.pepr.peprVersion, cfg.description, includedFiles);
+          execSync(`docker build --tag ${image} -f Dockerfile.controller .`, { stdio: "inherit" });
+          execSync(`docker push ${image}`, { stdio: "inherit" });
+        }
+      }
+
+      // If building without embedding, exit after building
+      if (!opts.embed) {
+        console.info(`✅ Module built successfully at ${path}`);
+        return;
+      }
+
+      // set the image version if provided
+      if (opts.version) {
+        cfg.pepr.peprVersion = opts.version;
+      }
+
+      // Generate a secret for the module
+      const assets = new Assets(
+        {
+          ...cfg.pepr,
+          appVersion: cfg.version,
+          description: cfg.description,
+        },
+        path,
+      );
+
+      // If registry is set to Iron Bank, use Iron Bank image
+      if (opts?.registry == "Iron Bank") {
+        console.warn(
+          `\n\tThis command assumes the latest release. Pepr's Iron Bank image release cycle is dictated by renovate and is typically released a few days after the GitHub release.\n\tAs an alternative you may consider custom --custom-image to target a specific image and version.`,
+        );
+        image = `registry1.dso.mil/ironbank/opensource/defenseunicorns/pepr/controller:v${cfg.pepr.peprVersion}`;
+      }
+
+      // if image is a custom image, use that instead of the default
+      if (image !== "") {
+        assets.image = image;
+      }
+
+      // Ensure imagePullSecret is valid
+      if (opts.withPullSecret) {
+        if (sanitizeResourceName(opts.withPullSecret) !== opts.withPullSecret) {
+          // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+          console.error(
+            "Invalid imagePullSecret. Please provide a valid name as defined in RFC 1123.",
+          );
+          process.exit(1);
+        }
+      }
+
+      const yamlFile = `pepr-module-${uuid}.yaml`;
+      const chartPath = `${uuid}-chart`;
+      const yamlPath = resolve(outputDir, yamlFile);
+      const yaml = await assets.allYaml(opts.rbacMode, opts.withPullSecret);
+
+      try {
+        // wait for capabilities to be loaded and test names
+        validateCapabilityNames(assets.capabilities);
+      } catch (e) {
+        console.error(`Error loading capability:`, e);
+        process.exit(1);
+      }
+
+      const zarfPath = resolve(outputDir, "zarf.yaml");
+
+      let zarf = "";
+      if (opts.zarf === "chart") {
+        zarf = assets.zarfYamlChart(chartPath);
+      } else {
+        zarf = assets.zarfYaml(yamlFile);
+      }
+      await fs.writeFile(yamlPath, yaml);
+      await fs.writeFile(zarfPath, zarf);
+
+      await assets.generateHelmChart(outputDir);
+
+      console.info(`✅ K8s resource for the module saved to ${yamlPath}`);
+    });
+}
+
+// Create a list of external libraries to exclude from the bundle, these are already stored in the container
+const externalLibs = Object.keys(dependencies);
+
+// Add the pepr library to the list of external libraries
+externalLibs.push("pepr");
+
+// Add the kubernetes client to the list of external libraries as it is pulled in by kubernetes-fluent-client
+externalLibs.push("@kubernetes/client-node");
+
+export async function loadModule(entryPoint = peprTS) {
+  // Resolve path to the module / files
+  const entryPointPath = resolve(".", entryPoint);
+  const modulePath = dirname(entryPointPath);
+  const cfgPath = resolve(modulePath, "package.json");
+
+  // Ensure the module's package.json and entrypoint files exist
+  try {
+    await fs.access(cfgPath);
+    await fs.access(entryPointPath);
+  } catch (e) {
+    console.error(
+      `Could not find ${cfgPath} or ${entryPointPath} in the current directory. Please run this command from the root of your module's directory.`,
+    );
+    process.exit(1);
+  }
+
+  // Read the module's UUID from the package.json file
+  const moduleText = await fs.readFile(cfgPath, { encoding: "utf-8" });
+  const cfg = JSON.parse(moduleText);
+  const { uuid } = cfg.pepr;
+  const name = `pepr-${uuid}.js`;
+
+  // Set the Pepr version from the current running version
+  cfg.pepr.peprVersion = version;
+
+  // Exit if the module's UUID could not be found
+  if (!uuid) {
+    throw new Error("Could not load the uuid in package.json");
+  }
+
+  return {
+    cfg,
+    entryPointPath,
+    modulePath,
+    name,
+    path: resolve(outputDir, name),
+    uuid,
+  };
+}
+
+export async function buildModule(reloader?: Reloader, entryPoint = peprTS, embed = true) {
+  try {
+    const { cfg, modulePath, path, uuid } = await loadModule(entryPoint);
+
+    const validFormat = await peprFormat(true);
+
+    if (!validFormat) {
+      console.log(
+        "\x1b[33m%s\x1b[0m",
+        "Formatting errors were found. The build will continue, but you may want to run `npx pepr format` to address any issues.",
+      );
+    }
+
+    // Resolve node_modules folder (in support of npm workspaces!)
+    const npmRoot = execFileSync("npm", ["root"]).toString().trim();
+
+    // Run `tsc` to validate the module's types & output sourcemaps
+    const args = ["--project", `${modulePath}/tsconfig.json`, "--outdir", outputDir];
+    execFileSync(`${npmRoot}/.bin/tsc`, args);
+
+    // Common build options for all builds
+    const ctxCfg: BuildOptions = {
+      bundle: true,
+      entryPoints: [entryPoint],
+      external: externalLibs,
+      format: "cjs",
+      keepNames: true,
+      legalComments: "external",
+      metafile: true,
+      minify: true,
+      outfile: path,
+      plugins: [
+        {
+          name: "reload-server",
+          setup(build) {
+            build.onEnd(async r => {
+              // Print the build size analysis
+              if (r?.metafile) {
+                console.log(await analyzeMetafile(r.metafile));
+              }
+
+              // If we're in dev mode, call the reloader function
+              if (reloader) {
+                await reloader(r);
+              }
+            });
+          },
+        },
+      ],
+      platform: "node",
+      sourcemap: true,
+      treeShaking: true,
+    };
+
+    if (reloader) {
+      // Only minify the code if we're not in dev mode
+      ctxCfg.minify = false;
+    }
+
+    // If not embedding (i.e. making a library module to be distro'd via NPM)
+    if (!embed) {
+      // Don't minify
+      ctxCfg.minify = false;
+
+      // Preserve the original file name
+      ctxCfg.outfile = resolve(outputDir, basename(entryPoint, extname(entryPoint))) + ".js";
+
+      // Don't bundle
+      ctxCfg.packages = "external";
+
+      // Don't tree shake
+      ctxCfg.treeShaking = false;
+    }
+
+    const ctx = await context(ctxCfg);
+
+    // If the reloader function is defined, watch the module for changes
+    if (reloader) {
+      await ctx.watch();
+    } else {
+      // Otherwise, just build the module once
+      await ctx.rebuild();
+      await ctx.dispose();
+    }
+
+    return { ctx, path, cfg, uuid };
+  } catch (e) {
+    console.error(`Error building module:`, e);
+
+    if (e.stdout) {
+      const out = e.stdout.toString() as string;
+      const err = e.stderr.toString();
+
+      console.log(out);
+      console.error(err);
+
+      // Check for version conflicts
+      if (out.includes("Types have separate declarations of a private property '_name'.")) {
+        // Try to find the conflicting package
+        const pgkErrMatch = /error TS2322: .*? 'import\("\/.*?\/node_modules\/(.*?)\/node_modules/g;
+        out.matchAll(pgkErrMatch);
+
+        // Look for package conflict errors
+        const conflicts = [...out.matchAll(pgkErrMatch)];
+
+        // If the regex didn't match, leave a generic error
+        if (conflicts.length < 1) {
+          console.warn(
+            `\n\tOne or more imported Pepr Capabilities seem to be using an incompatible version of Pepr.\n\tTry updating your Pepr Capabilities to their latest versions.`,
+            "Version Conflict",
+          );
+        }
+
+        // Otherwise, loop through each conflicting package and print an error
+        conflicts.forEach(match => {
+          console.warn(
+            `\n\tPackage '${match[1]}' seems to be incompatible with your current version of Pepr.\n\tTry updating to the latest version.`,
+            "Version Conflict",
+          );
+        });
+      }
+    }
+
+    // On any other error, exit with a non-zero exit code
+    process.exit(1);
+  }
+}

package/src/cli/deploy.ts

@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
+
+import prompt from "prompts";
+
+import { Assets } from "../lib/assets";
+import { buildModule } from "./build";
+import { RootCmd } from "./root";
+import { validateCapabilityNames, namespaceDeploymentsReady } from "../lib/helpers";
+import { ImagePullSecret } from "../lib/types";
+import { sanitizeName } from "./init/utils";
+import { deployImagePullSecret } from "../lib/assets/deploy";
+
+export default function (program: RootCmd) {
+  program
+    .command("deploy")
+    .description("Deploy a Pepr Module")
+    .option("-i, --image [image]", "Override the image tag")
+    .option("--confirm", "Skip confirmation prompt")
+    .option("--pullSecret <name>", "Deploy imagePullSecret for Controller private registry")
+    .option("--docker-server <server>", "Docker server address")
+    .option("--docker-username <username>", "Docker registry username")
+    .option("--docker-email <email>", "Email for Docker registry")
+    .option("--docker-password <password>", "Password for Docker registry")
+    .option("--force", "Force deploy the module, override manager field")
+    .action(async opts => {
+      let imagePullSecret: ImagePullSecret | undefined;
+
+      if (
+        opts.pullSecret &&
+        opts.pullSecret.length > 0 &&
+        (!opts.dockerServer || !opts.dockerUsername || !opts.dockerEmail || !opts.dockerPassword)
+      ) {
+        console.error(
+          "Error: Must provide docker server, username, email, and password when providing pull secret",
+        );
+        process.exit(1);
+      } else if (opts.pullSecret && opts.pullSecret !== sanitizeName(opts.pullSecret)) {
+        // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+        console.error(
+          "Invalid imagePullSecret name. Please provide a valid name as defined in RFC 1123.",
+        );
+        process.exit(1);
+      } else if (opts.pullSecret) {
+        imagePullSecret = {
+          auths: {
+            [opts.dockerServer]: {
+              username: opts.dockerUsername,
+              password: opts.dockerPassword,
+              email: opts.dockerEmail,
+              auth: Buffer.from(`${opts.dockerUsername}:${opts.dockerPassword}`).toString("base64"),
+            },
+          },
+        };
+
+        await deployImagePullSecret(imagePullSecret, opts.pullSecret);
+        return;
+      }
+
+      if (!opts.confirm) {
+        // Prompt the user to confirm
+        const confirm = await prompt({
+          type: "confirm",
+          name: "confirm",
+          message: "This will remove and redeploy the module. Continue?",
+        });
+
+        // Exit if the user doesn't confirm
+        if (!confirm.confirm) {
+          process.exit(0);
+        }
+      }
+
+      // Build the module
+      const { cfg, path } = await buildModule();
+
+      // Generate a secret for the module
+      const webhook = new Assets(
+        {
+          ...cfg.pepr,
+          description: cfg.description,
+        },
+        path,
+      );
+
+      if (opts.image) {
+        webhook.image = opts.image;
+      }
+
+      // Identify conf'd webhookTimeout to give to deploy call
+      const timeout = cfg.pepr.webhookTimeout ? cfg.pepr.webhookTimeout : 10;
+
+      try {
+        await webhook.deploy(opts.force, timeout);
+        // wait for capabilities to be loaded and test names
+        validateCapabilityNames(webhook.capabilities);
+        // Wait for the pepr-system resources to be fully up
+        await namespaceDeploymentsReady();
+        console.info(`✅ Module deployed successfully`);
+      } catch (e) {
+        console.error(`Error deploying module:`, e);
+        process.exit(1);
+      }
+    });
+}

package/src/cli/dev.ts

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
+
+import { ChildProcess, fork } from "child_process";
+import { promises as fs } from "fs";
+import prompt from "prompts";
+import { validateCapabilityNames } from "../lib/helpers";
+import { Assets } from "../lib/assets";
+import { buildModule, loadModule } from "./build";
+import { RootCmd } from "./root";
+import { K8s, kind } from "kubernetes-fluent-client";
+import { PeprStore } from "../lib/k8s";
+export default function (program: RootCmd) {
+  program
+    .command("dev")
+    .description("Setup a local webhook development environment")
+    .option("-h, --host [host]", "Host to listen on", "host.k3d.internal")
+    .option("--confirm", "Skip confirmation prompt")
+    .action(async opts => {
+      // Prompt the user to confirm if they didn't pass the --confirm flag
+      if (!opts.confirm) {
+        const confirm = await prompt({
+          type: "confirm",
+          name: "confirm",
+          message: "This will remove and redeploy the module. Continue?",
+        });
+
+        // Exit if the user doesn't confirm
+        if (!confirm.confirm) {
+          process.exit(0);
+        }
+      }
+
+      // Build the module
+      const { cfg, path } = await loadModule();
+
+      // Generate a secret for the module
+      const webhook = new Assets(
+        {
+          ...cfg.pepr,
+          description: cfg.description,
+        },
+        path,
+        opts.host,
+      );
+
+      // Write the TLS cert and key to disk
+      await fs.writeFile("insecure-tls.crt", webhook.tls.pem.crt);
+      await fs.writeFile("insecure-tls.key", webhook.tls.pem.key);
+
+      try {
+        let program: ChildProcess;
+        const name = `pepr-${cfg.pepr.uuid}`;
+        const scheduleStore = `pepr-${cfg.pepr.uuid}-schedule`;
+        const store = `pepr-${cfg.pepr.uuid}-store`;
+
+        // Run the processed javascript file
+        const runFork = async () => {
+          console.info(`Running module ${path}`);
+
+          // Deploy the webhook with a 30 second timeout for debugging, don't force
+          await webhook.deploy(false, 30);
+
+          try {
+            // wait for capabilities to be loaded and test names
+            validateCapabilityNames(webhook.capabilities);
+          } catch (e) {
+            console.error(`Error validating capability names:`, e);
+            process.exit(1);
+          }
+
+          program = fork(path, {
+            env: {
+              ...process.env,
+              LOG_LEVEL: "debug",
+              PEPR_MODE: "dev",
+              PEPR_API_TOKEN: webhook.apiToken,
+              PEPR_PRETTY_LOGS: "true",
+              SSL_KEY_PATH: "insecure-tls.key",
+              SSL_CERT_PATH: "insecure-tls.crt",
+            },
+            stdio: "inherit",
+          });
+
+          program.on("close", async () => {
+            await Promise.all([
+              K8s(kind.MutatingWebhookConfiguration).Delete(name),
+              K8s(kind.ValidatingWebhookConfiguration).Delete(name),
+              K8s(PeprStore).InNamespace("pepr-system").Delete(scheduleStore),
+              K8s(PeprStore).InNamespace("pepr-system").Delete(store),
+            ]);
+          });
+
+          // listen for CTRL+C and remove webhooks
+          process.on("SIGINT", () => {
+            console.debug(`Received SIGINT, removing webhooks`);
+          });
+        };
+
+        await buildModule(async r => {
+          if (r.errors.length > 0) {
+            console.error(`Error compiling module: ${r.errors}`);
+            return;
+          }
+
+          if (program) {
+            program.once("exit", runFork);
+            program.kill("SIGKILL");
+          } else {
+            await runFork();
+          }
+        });
+      } catch (e) {
+        console.error(`Error deploying module:`, e);
+        process.exit(1);
+      }
+    });
+}

package/src/cli/format.ts

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The Pepr Authors
+
+import { ESLint } from "eslint";
+import { promises as fs } from "fs";
+import { format, resolveConfig } from "prettier";
+
+import { RootCmd } from "./root";
+
+export default function (program: RootCmd) {
+  program
+    .command("format")
+    .description("Lint and format this Pepr module")
+    .option("-v, --validate-only", "Do not modify files, only validate formatting")
+    .action(async opts => {
+      const success = await peprFormat(opts.validateOnly);
+
+      if (success) {
+        console.info("✅ Module formatted");
+      } else {
+        process.exit(1);
+      }
+    });
+}
+
+/**
+ * Perform linting and formatting on the module
+ * @param validateOnly
+ * @returns success
+ */
+export async function peprFormat(validateOnly: boolean) {
+  {
+    try {
+      const eslint = new ESLint();
+      const results = await eslint.lintFiles(["./**/*.ts"]);
+
+      // Track if any files failed
+      let hasFailure = false;
+
+      results.forEach(async result => {
+        const errorCount = result.fatalErrorCount + result.errorCount;
+        if (errorCount > 0) {
+          hasFailure = true;
+        }
+      });
+
+      const formatter = await eslint.loadFormatter("stylish");
+      const resultText = await formatter.format(results, {} as ESLint.LintResultData);
+
+      if (resultText) {
+        console.log(resultText);
+      }
+
+      // Write the fixes if not in validate-only mode
+      if (!validateOnly) {
+        await ESLint.outputFixes(results);
+      }
+
+      // Format with Prettier
+      for (const { filePath } of results) {
+        const content = await fs.readFile(filePath, "utf8");
+        const cfg = await resolveConfig(filePath);
+        const formatted = await format(content, { filepath: filePath, ...cfg });
+
+        // If in validate-only mode, check if the file is formatted correctly
+        if (validateOnly) {
+          if (formatted !== content) {
+            hasFailure = true;
+            console.error(`File ${filePath} is not formatted correctly`);
+          }
+        } else {
+          // Otherwise, write the formatted file
+          await fs.writeFile(filePath, formatted);
+        }
+      }
+
+      return !hasFailure;
+    } catch (e) {
+      console.error(`Error formatting module:`, e);
+      return false;
+    }
+  }
+}