underpost 2.95.3 → 2.96.0

package/src/cli/baremetal.js

@@ -4,15 +4,19 @@
  * @namespace UnderpostBaremetal
  */
 
+import { fileURLToPath } from 'url';
 import { getNpmRootPath, getUnderpostRootPath } from '../server/conf.js';
 import { openTerminal, pbcopy, shellExec } from '../server/process.js';
 import dotenv from 'dotenv';
 import { loggerFactory } from '../server/logger.js';
 import { getLocalIPv4Address } from '../server/dns.js';
 import fs from 'fs-extra';
+import path from 'path';
 import Downloader from '../server/downloader.js';
 import UnderpostCloudInit from './cloud-init.js';
+import UnderpostRepository from './repository.js';
 import { s4, timer } from '../client/components/core/CommonJs.js';
+import { spawnSync } from 'child_process';
 
 const logger = loggerFactory(import.meta);
 
@@ -25,6 +29,19 @@ const logger = loggerFactory(import.meta);
  */
 class UnderpostBaremetal {
   static API = {
+    /**
+     * @method installPacker
+     * @description Installs Packer CLI.
+     * @memberof UnderpostBaremetal
+     * @returns {Promise<void>}
+     */
+    async installPacker(underpostRoot) {
+      const scriptPath = `${underpostRoot}/scripts/packer-setup.sh`;
+      logger.info(`Installing Packer using script: ${scriptPath}`);
+      shellExec(`sudo chmod +x ${scriptPath}`);
+      shellExec(`sudo ${scriptPath}`);
+    },
+
     /**
      * @method callback
      * @description Initiates a baremetal provisioning workflow based on the provided options.
@@ -40,6 +57,12 @@ class UnderpostBaremetal {
      * @param {boolean} [options.controlServerUninstall=false] - Flag to uninstall the control server.
      * @param {boolean} [options.controlServerDbInstall=false] - Flag to install the control server's database.
      * @param {boolean} [options.controlServerDbUninstall=false] - Flag to uninstall the control server's database.
+     * @param {boolean} [options.installPacker=false] - Flag to install Packer CLI.
+     * @param {string} [options.packerMaasImageTemplate] - Template path from canonical/packer-maas to extract (requires workflow-id).
+     * @param {string} [options.packerWorkflowId] - Workflow ID for Packer MAAS image operations (used with --packer-maas-image-build or --packer-maas-image-upload).
+     * @param {boolean} [options.packerMaasImageBuild=false] - Flag to build a Packer MAAS image for the workflow specified by packerWorkflowId.
+     * @param {boolean} [options.packerMaasImageUpload=false] - Flag to upload a Packer MAAS image artifact without rebuilding for the workflow specified by packerWorkflowId.
+     * @param {boolean} [options.cloudInitUpdate=false] - Flag to update cloud-init configuration on the baremetal machine.
      * @param {boolean} [options.commission=false] - Flag to commission the baremetal machine.
      * @param {boolean} [options.nfsBuild=false] - Flag to build the NFS root filesystem.
      * @param {boolean} [options.nfsMount=false] - Flag to mount the NFS root filesystem.
@@ -59,6 +82,12 @@ class UnderpostBaremetal {
         controlServerUninstall: false,
         controlServerDbInstall: false,
         controlServerDbUninstall: false,
+        installPacker: false,
+        packerMaasImageTemplate: false,
+        packerWorkflowId: '',
+        packerMaasImageBuild: false,
+        packerMaasImageUpload: false,
+        cloudInitUpdate: false,
         commission: false,
         nfsBuild: false,
         nfsMount: false,
@@ -106,6 +135,164 @@ class UnderpostBaremetal {
       // Log the initiation of the baremetal callback with relevant metadata.
       logger.info('Baremetal callback', callbackMetaData);
 
+      if (options.installPacker) {
+        await UnderpostBaremetal.API.installPacker(underpostRoot);
+        return;
+      }
+
+      if (options.packerMaasImageTemplate) {
+        if (!workflowId) {
+          throw new Error('workflow-id is required when using --packer-maas-image-template');
+        }
+
+        const templatePath = options.packerMaasImageTemplate;
+        const targetDir = `${underpostRoot}/packer/images/${workflowId}`;
+
+        logger.info(`Creating new Packer MAAS image template for workflow: ${workflowId}`);
+        logger.info(`Template path: ${templatePath}`);
+        logger.info(`Target directory: ${targetDir}`);
+
+        try {
+          // Use UnderpostRepository to copy files from GitHub
+          const result = await UnderpostRepository.API.copyGitUrlDirectoryRecursive({
+            gitUrl: 'https://github.com/canonical/packer-maas',
+            directoryPath: templatePath,
+            targetPath: targetDir,
+            branch: 'main',
+            overwrite: false,
+          });
+
+          logger.info(`\nSuccessfully copied ${result.filesCount} files`);
+
+          // Create empty workflow configuration entry
+          const workflowConfig = {
+            dir: `packer/images/${workflowId}`,
+            maas: {
+              name: `custom/${workflowId.toLowerCase()}`,
+              title: `${workflowId} Custom`,
+              architecture: 'amd64/generic',
+              base_image: 'ubuntu/22.04',
+              filetype: 'tgz',
+              content: `${workflowId.toLowerCase()}.tar.gz`,
+            },
+          };
+
+          const workflows = UnderpostBaremetal.API.loadPackerMaasImageBuildWorkflows();
+          workflows[workflowId] = workflowConfig;
+          UnderpostBaremetal.API.writePackerMaasImageBuildWorkflows(workflows);
+
+          logger.info('\nTemplate extracted successfully!');
+          logger.info(`\nAdded configuration for ${workflowId} to engine/baremetal/packer-workflows.json`);
+          logger.info('\nNext steps:');
+          logger.info(`1. Review and customize the Packer template files in: ${targetDir}`);
+          logger.info(`2. Review the workflow configuration in engine/baremetal/packer-workflows.json`);
+          logger.info(
+            `3. Build the image with: underpost baremetal ${workflowId} --packer-maas-image-build ${workflowId}`,
+          );
+        } catch (error) {
+          throw new Error(`Failed to extract template: ${error.message}`);
+        }
+
+        return;
+      }
+
+      if (options.packerMaasImageBuild || options.packerMaasImageUpload) {
+        // Use the workflow ID from --packer-workflow-id option
+        if (!options.packerWorkflowId) {
+          throw new Error('Workflow ID is required. Please specify using --packer-workflow-id <workflow-id>');
+        }
+
+        workflowId = options.packerWorkflowId;
+
+        const workflow = UnderpostBaremetal.API.loadPackerMaasImageBuildWorkflows()[workflowId];
+        if (!workflow) {
+          throw new Error(`Packer MAAS image build workflow not found: ${workflowId}`);
+        }
+        const packerDir = `${underpostRoot}/${workflow.dir}`;
+        const tarballPath = `${packerDir}/${workflow.maas.content}`;
+
+        // Build phase (skip if upload-only mode)
+        if (options.packerMaasImageBuild) {
+          if (shellExec('packer version', { silent: true }).code !== 0) {
+            throw new Error('Packer is not installed. Please install Packer to proceed.');
+          }
+
+          logger.info(`Building Packer image for ${workflowId} in ${packerDir}...`);
+          const artifacts = ['output-rocky9', 'packer_cache', 'x86_64_VARS.fd', 'rocky9.tar.gz'];
+          shellExec(`cd packer/images/${workflowId}
+rm -rf ${artifacts.join(' ')}`);
+          shellExec(`chmod +x ${underpostRoot}/scripts/packer-init-vars-file.sh`);
+          shellExec(`${underpostRoot}/scripts/packer-init-vars-file.sh`);
+
+          const init = spawnSync('packer', ['init', '.'], { stdio: 'inherit', cwd: packerDir });
+          if (init.status !== 0) {
+            throw new Error('Packer init failed');
+          }
+
+          const build = spawnSync('packer', ['build', '.'], {
+            stdio: 'inherit',
+            cwd: packerDir,
+            env: { ...process.env, PACKER_LOG: '1' },
+          });
+
+          if (build.status !== 0) {
+            throw new Error('Packer build failed');
+          }
+        } else {
+          // Upload-only mode: verify tarball exists
+          logger.info(`Upload-only mode: checking for existing build artifact...`);
+          if (!fs.existsSync(tarballPath)) {
+            throw new Error(
+              `Build artifact not found: ${tarballPath}\n` +
+                `Please build first with: --packer-maas-image-build ${workflowId}`,
+            );
+          }
+          const stats = fs.statSync(tarballPath);
+          logger.info(`Found existing artifact: ${tarballPath} (${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB)`);
+        }
+
+        logger.info(`Uploading image to MAAS...`);
+
+        // Detect MAAS profile from 'maas list' output
+        let maasProfile = process.env.MAAS_ADMIN_USERNAME;
+        if (!maasProfile) {
+          const profileList = shellExec('maas list', { silent: true, stdout: true });
+          if (profileList) {
+            const firstLine = profileList.trim().split('\n')[0];
+            const match = firstLine.match(/^(\S+)\s+http/);
+            if (match) {
+              maasProfile = match[1];
+              logger.info(`Detected MAAS profile: ${maasProfile}`);
+            }
+          }
+        }
+
+        if (!maasProfile) {
+          throw new Error(
+            'MAAS profile not found. Please run "maas login" first or set MAAS_ADMIN_USERNAME environment variable.',
+          );
+        }
+
+        // Use the upload script to avoid MAAS CLI bugs
+        const uploadScript = `${underpostRoot}/scripts/maas-upload-boot-resource.sh`;
+        const uploadCmd = `${uploadScript} ${maasProfile} "${workflow.maas.name}" "${workflow.maas.title}" "${workflow.maas.architecture}" "${workflow.maas.base_image}" "${workflow.maas.filetype}" "${tarballPath}"`;
+
+        logger.info(`Uploading to MAAS using: ${uploadScript}`);
+        const uploadResult = shellExec(uploadCmd);
+        if (uploadResult.code !== 0) {
+          logger.error(`Upload failed with exit code: ${uploadResult.code}`);
+          if (uploadResult.stdout) {
+            logger.error(`Upload output:\n${uploadResult.stdout}`);
+          }
+          if (uploadResult.stderr) {
+            logger.error(`Upload error output:\n${uploadResult.stderr}`);
+          }
+          throw new Error('MAAS upload failed - see output above for details');
+        }
+        logger.info(`Successfully uploaded ${workflow.maas.name} to MAAS!`);
+        return;
+      }
+
       // Handle various log display options.
       if (options.logs === 'dhcp') {
         shellExec(`journalctl -f -t dhcpd -u snap.maas.pebble.service`);
@@ -129,7 +316,11 @@ class UnderpostBaremetal {
 
       // Handle NFS shell access option.
       if (options.nfsSh === true) {
-        const { debootstrap } = UnderpostBaremetal.API.workflowsConfig[workflowId];
+        const workflowsConfig = UnderpostBaremetal.API.loadWorkflowsConfig();
+        if (!workflowsConfig[workflowId]) {
+          throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
+        }
+        const { debootstrap } = workflowsConfig[workflowId];
         // Copy the chroot command to the clipboard for easy execution.
         if (debootstrap.image.architecture !== callbackMetaData.runnerHost.architecture)
           switch (debootstrap.image.architecture) {
@@ -194,9 +385,14 @@ class UnderpostBaremetal {
         return;
       }
 
+      const workflowsConfig = UnderpostBaremetal.API.loadWorkflowsConfig();
+      if (!workflowsConfig[workflowId]) {
+        throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
+      }
+
       // Set debootstrap architecture.
       {
-        const { architecture } = UnderpostBaremetal.API.workflowsConfig[workflowId].debootstrap.image;
+        const { architecture } = workflowsConfig[workflowId].debootstrap.image;
         debootstrapArch = architecture;
       }
 
@@ -216,11 +412,6 @@ class UnderpostBaremetal {
       if (options.nfsBuild === true) {
         // Check if NFS is already mounted to avoid redundant builds.
         const { isMounted } = UnderpostBaremetal.API.nfsMountCallback({ hostname, workflowId });
-        if (isMounted) {
-          logger.warn('NFS root filesystem is mounted, skipping build.');
-          return; // Exit if already mounted.
-        }
-        logger.info('NFS root filesystem is not mounted, building...');
 
         // Clean and create the NFS host path.
         shellExec(`sudo rm -rf ${nfsHostPath}/*`);
@@ -231,7 +422,7 @@ class UnderpostBaremetal {
 
         // Perform the first stage of debootstrap.
         {
-          const { architecture, name } = UnderpostBaremetal.API.workflowsConfig[workflowId].debootstrap.image;
+          const { architecture, name } = workflowsConfig[workflowId].debootstrap.image;
           shellExec(
             [
               `sudo debootstrap`,
@@ -276,7 +467,7 @@ class UnderpostBaremetal {
 
         // Apply system provisioning steps (base, user, timezone, keyboard).
         {
-          const { systemProvisioning, kernelLibVersion, chronyc } = UnderpostBaremetal.API.workflowsConfig[workflowId];
+          const { systemProvisioning, kernelLibVersion, chronyc } = workflowsConfig[workflowId];
           const { timezone, chronyConfPath } = chronyc;
 
           UnderpostBaremetal.API.crossArchRunner({
@@ -330,8 +521,7 @@ class UnderpostBaremetal {
 
       // Handle commissioning tasks (placeholder for future implementation).
       if (options.commission === true) {
-        const { firmwares, networkInterfaceName, maas, netmask, menuentryStr } =
-          UnderpostBaremetal.API.workflowsConfig[workflowId];
+        const { firmwares, networkInterfaceName, maas, netmask, menuentryStr } = workflowsConfig[workflowId];
         const resource = resources.find(
           (o) => o.architecture === maas.image.architecture && o.name === maas.image.name,
         );
@@ -493,7 +683,7 @@ menuentry '${menuentryStr}' {
 
       // Final commissioning steps.
       if (options.commission === true || options.cloudInitUpdate === true) {
-        const { debootstrap, networkInterfaceName, chronyc, maas } = UnderpostBaremetal.API.workflowsConfig[workflowId];
+        const { debootstrap, networkInterfaceName, chronyc, maas } = workflowsConfig[workflowId];
         const { timezone, chronyConfPath } = chronyc;
 
         // Build cloud-init tools.
@@ -630,8 +820,13 @@ menuentry '${menuentryStr}' {
           const machine = {
             architecture: maas.image.architecture.match('amd') ? 'amd64/generic' : 'arm64/generic',
             mac_address: discovery.mac_address,
-            hostname:
-              discovery.hostname ?? discovery.mac_organization ?? discovery.domain ?? `generic-host-${s4()}${s4()}`,
+            hostname: discovery.hostname
+              ? discovery.hostname
+              : discovery.mac_organization
+                ? discovery.mac_organization
+                : discovery.domain
+                  ? discovery.domain
+                  : `generic-host-${s4()}${s4()}`,
             power_type: 'manual',
             mac_addresses: discovery.mac_address,
             ip: discovery.ip,
@@ -740,9 +935,9 @@ menuentry '${menuentryStr}' {
       // Install necessary packages for debootstrap and QEMU.
       shellExec(`sudo dnf install -y iptables-legacy`);
       shellExec(`sudo dnf install -y debootstrap`);
-      shellExec(`sudo dnf install kernel-modules-extra-$(uname -r)`);
+      shellExec(`sudo dnf install -y kernel-modules-extra-$(uname -r)`);
       // Reset QEMU user-static binfmt for proper cross-architecture execution.
-      shellExec(`sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes`);
+      shellExec(`sudo podman run --rm --privileged docker.io/multiarch/qemu-user-static:latest --reset -p yes`);
       // Mount binfmt_misc filesystem.
       shellExec(`sudo modprobe binfmt_misc`);
       shellExec(`sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc`);
@@ -826,8 +1021,8 @@ menuentry '${menuentryStr}' {
           break;
       }
       // Install GRUB EFI modules for both architectures to ensure compatibility.
-      shellExec(`sudo dnf install grub2-efi-aa64-modules`);
-      shellExec(`sudo dnf install grub2-efi-x64-modules`);
+      shellExec(`sudo dnf install -y grub2-efi-aa64-modules`);
+      shellExec(`sudo dnf install -y grub2-efi-x64-modules`);
     },
 
     /**
@@ -912,9 +1107,13 @@ EOF`);
      */
     nfsMountCallback({ hostname, workflowId, mount, unmount }) {
       let isMounted = false;
+      const workflowsConfig = UnderpostBaremetal.API.loadWorkflowsConfig();
+      if (!workflowsConfig[workflowId]) {
+        throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
+      }
       // Iterate through defined NFS mounts in the workflow configuration.
-      for (const mountCmd of Object.keys(UnderpostBaremetal.API.workflowsConfig[workflowId].nfs.mounts)) {
-        for (const mountPath of UnderpostBaremetal.API.workflowsConfig[workflowId].nfs.mounts[mountCmd]) {
+      for (const mountCmd of Object.keys(workflowsConfig[workflowId].nfs.mounts)) {
+        for (const mountPath of workflowsConfig[workflowId].nfs.mounts[mountCmd]) {
           const hostMountPath = `${process.env.NFS_EXPORT_PATH}/${hostname}${mountPath}`;
           // Check if the path is already mounted using `mountpoint` command.
           const isPathMounted = !shellExec(`mountpoint ${hostMountPath}`, { silent: true, stdout: true }).match(
@@ -1044,6 +1243,8 @@ SOURCES`,
           `export DEBIAN_FRONTEND=noninteractive`, // Set non-interactive mode for Debian packages.
           `ln -fs /usr/share/zoneinfo/${timezone} /etc/localtime`, // Symlink timezone.
           `sudo dpkg-reconfigure --frontend noninteractive tzdata`, // Reconfigure timezone data.
+          `sudo timedatectl set-timezone ${timezone}`, // Set timezone using timedatectl.
+          `sudo timedatectl set-ntp true`, // Enable NTP synchronization.
 
           // Write the Chrony configuration file.
           `echo '
@@ -1092,6 +1293,11 @@ logdir /var/log/chrony
           // Enable, restart, and check status of Chrony service.
           `sudo systemctl enable --now ${alias}`,
           `sudo systemctl restart ${alias}`,
+
+          // Wait for chrony to synchronize
+          `echo "Waiting for chrony to synchronize..."`,
+          `for i in {1..30}; do chronyc tracking | grep -q "Leap status     : Normal" && break || sleep 2; done`,
+
           `sudo systemctl status ${alias}`,
 
           // Verify Chrony synchronization.
@@ -1253,55 +1459,46 @@ GATEWAY=${gateway}`;
     },
 
     /**
-     * @property {object} workflowsConfig
-     * @description Configuration for different baremetal provisioning workflows.
+     * @method loadWorkflowsConfig
+     * @namespace UnderpostBaremetal.API
+     * @description Loads the commission workflows configuration from commission-workflows.json.
      * Each workflow defines specific parameters like system provisioning type,
      * kernel version, Chrony settings, debootstrap image details, and NFS mounts.     *
      * @memberof UnderpostBaremetal
      */
-    workflowsConfig: {
-      /**
-       * @property {object} rpi4mb
-       * @description Configuration for the Raspberry Pi 4 Model B workflow.
-       * @memberof UnderpostBaremetal.workflowsConfig
-       */
-      rpi4mb: {
-        menuentryStr: 'UNDERPOST.NET UEFI/GRUB/MAAS RPi4 commissioning (ARM64)',
-        systemProvisioning: 'ubuntu', // Specifies the system provisioning factory to use.
-        kernelLibVersion: `6.8.0-41-generic`, // The kernel library version for this workflow.
-        networkInterfaceName: 'enabcm6e4ei0', // The name of the primary network interface on the RPi4.
-        netmask: '255.255.255.0', // Subnet mask for the network.
-        firmwares: [
-          {
-            url: 'https://github.com/pftf/RPi4/releases/download/v1.41/RPi4_UEFI_Firmware_v1.41.zip',
-            gateway: '192.168.1.1',
-            subnet: '255.255.255.0',
-          },
-        ],
-        chronyc: {
-          timezone: 'America/New_York', // Timezone for Chrony configuration.
-          chronyConfPath: `/etc/chrony/chrony.conf`, // Path to Chrony configuration file.
-        },
-        debootstrap: {
-          image: {
-            architecture: 'arm64', // Architecture for the debootstrap image.
-            name: 'noble', // Codename of the Ubuntu release (e.g., 'noble' for 24.04 LTS).
-          },
-        },
-        maas: {
-          image: {
-            architecture: 'arm64/ga-24.04', // Architecture for MAAS image.
-            name: 'ubuntu/noble', // Name of the MAAS Ubuntu image.
-          },
-        },
-        nfs: {
-          mounts: {
-            // Define NFS mount points and their types (bind, rbind).
-            bind: ['/proc', '/sys', '/run'], // Standard bind mounts.
-            rbind: ['/dev'], // Recursive bind mount for /dev.
-          },
-        },
-      },
+    loadWorkflowsConfig() {
+      if (this._workflowsConfig) return this._workflowsConfig;
+      const __dirname = path.dirname(fileURLToPath(import.meta.url));
+      const configPath = path.resolve(__dirname, '../../baremetal/commission-workflows.json');
+      this._workflowsConfig = fs.readJsonSync(configPath);
+      return this._workflowsConfig;
+    },
+
+    /**
+     * @property {object} packerMaasImageBuildWorkflows
+     * @description Configuration for PACKe mass image workflows.
+     * @memberof UnderpostBaremetal
+     */
+    loadPackerMaasImageBuildWorkflows() {
+      if (this._packerMaasImageBuildWorkflows) return this._packerMaasImageBuildWorkflows;
+      const __dirname = path.dirname(fileURLToPath(import.meta.url));
+      const configPath = path.resolve(__dirname, '../../baremetal/packer-workflows.json');
+      this._packerMaasImageBuildWorkflows = fs.readJsonSync(configPath);
+      return this._packerMaasImageBuildWorkflows;
+    },
+
+    /**
+     * Write Packer MAAS image build workflows configuration to file
+     * @param {object} workflows - The workflows configuration object
+     * @description Writes the Packer MAAS image build workflows to packer-workflows.json
+     * @memberof UnderpostBaremetal
+     */
+    writePackerMaasImageBuildWorkflows(workflows) {
+      const __dirname = path.dirname(fileURLToPath(import.meta.url));
+      const configPath = path.resolve(__dirname, '../../baremetal/packer-workflows.json');
+      fs.writeJsonSync(configPath, workflows, { spaces: 2 });
+      this._packerMaasImageBuildWorkflows = workflows;
+      return configPath;
     },
   };
 }

package/src/cli/cloud-init.js

@@ -43,7 +43,7 @@ class UnderpostCloudInit {
     buildTools({ workflowId, nfsHostPath, hostname, callbackMetaData, dev }) {
       // Destructure workflow configuration for easier access.
       const { systemProvisioning, chronyc, networkInterfaceName, debootstrap } =
-        UnderpostBaremetal.API.workflowsConfig[workflowId];
+        UnderpostBaremetal.API.loadWorkflowsConfig()[workflowId];
       const { timezone, chronyConfPath } = chronyc;
       // Define the specific directory for underpost tools within the NFS host path.
       const nfsHostToolsPath = `${nfsHostPath}/underpost`;
@@ -159,12 +159,18 @@ echo "sudo cloud-init modules --mode=final"`,
           logger.info('Build', `${nfsHostToolsPath}/test.sh`);
           fs.writeFileSync(
             `${nfsHostToolsPath}/test.sh`,
-            `echo -e "\n=== Current date/time ==="
+            `echo -e "\necho -e "\n=== Registered users ==="
+cut -d: -f1 /etc/passwd
+=== Current date/time ==="
 date '+%Y-%m-%d %H:%M:%S'
+echo -e "\n=== Timezone Configuration ==="
+timedatectl status
+echo -e "\n=== Chrony Synchronization Status ==="
+chronyc tracking
+echo -e "\n=== Chrony Sources ==="
+chronyc sources
 echo -e "\n=== Keyboard layout ==="
-cat /etc/default/keyboard
-echo -e "\n=== Registered users ==="
-cut -d: -f1 /etc/passwd`,
+cat /etc/default/keyboard`,
             'utf8',
           );