underpost 2.8.652 → 2.8.781

package/bin/deploy.js

@@ -26,9 +26,10 @@ import {
   fixDependencies,
   setUpProxyMaintenanceServer,
   writeEnv,
+  getUnderpostRootPath,
 } from '../src/server/conf.js';
 import { buildClient } from '../src/server/client-build.js';
-import { range, setPad, timer, uniqueArray } from '../src/client/components/core/CommonJs.js';
+import { range, s4, setPad, timer, uniqueArray } from '../src/client/components/core/CommonJs.js';
 import { MongooseDB } from '../src/db/mongo/MongooseDB.js';
 import { Lampp } from '../src/runtime/lampp/Lampp.js';
 import { DefaultConf } from '../conf.js';
@@ -37,6 +38,11 @@ import { JSONweb } from '../src/server/client-formatted.js';
 import { Xampp } from '../src/runtime/xampp/Xampp.js';
 import { ejs } from '../src/server/json-schema.js';
 import { buildCliDoc } from '../src/cli/index.js';
+import { getLocalIPv4Address, ip } from '../src/server/dns.js';
+import { Downloader } from '../src/server/downloader.js';
+import colors from 'colors';
+
+colors.enable();
 
 const logger = loggerFactory(import.meta);
 
@@ -44,6 +50,82 @@ logger.info('argv', process.argv);
 
 const [exe, dir, operator] = process.argv;
 
+const updateVirtualRoot = async ({ nfsHostPath, IP_ADDRESS, ipaddr }) => {
+  const steps = [
+    `apt update`,
+    `ln -sf /lib/systemd/systemd /sbin/init`,
+    // `sudo apt install linux-modules-extra-6.8.0-31-generic`,
+    `apt install -y sudo`,
+    `apt install -y ntp`,
+    `apt install -y openssh-server`,
+    `apt install -y iptables`,
+    `update-alternatives --set iptables /usr/sbin/iptables-legacy`,
+    `update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy`,
+    `apt install -y locales`,
+    `apt install -y cloud-init`,
+    `mkdir -p /var/lib/cloud`,
+    `chown -R root:root /var/lib/cloud`,
+    `chmod -R 0755 /var/lib/cloud`,
+    `mkdir -p /home/root/.ssh`,
+    `echo '${fs.readFileSync(
+      `/home/dd/engine/engine-private/deploy/id_rsa.pub`,
+      'utf8',
+    )}' >> /home/root/.ssh/authorized_keys`,
+    `chmod 700 /home/root/.ssh`,
+    `chmod 600 /home/root/.ssh/authorized_keys`,
+    `systemctl enable ssh`,
+    `systemctl enable ntp`,
+    `apt install -y linux-generic-hwe-24.04`,
+    `modprobe ip_tables`,
+    `cat <<EOF_MAAS_CFG > /etc/cloud/cloud.cfg.d/90_maas.cfg
+datasource_list: [ MAAS ]
+datasource:
+  MAAS:
+    metadata_url: http://${IP_ADDRESS}:5248/MAAS/metadata
+users:
+  - name: ${process.env.MAAS_ADMIN_USERNAME}
+    ssh_authorized_keys:
+      - ${fs.readFileSync(`/home/dd/engine/engine-private/deploy/id_rsa.pub`, 'utf8')}
+    sudo: "ALL=(ALL) NOPASSWD:ALL"
+    groups: sudo
+    shell: /bin/bash
+packages:
+  - git
+  - htop
+  - ufw
+# package_update: true
+runcmd:
+  - ufw enable
+  - ufw allow ssh
+resize_rootfs: false
+growpart:
+  mode: off
+network:
+  version: 2
+  ethernets:
+    ${process.env.RPI4_INTERFACE_NAME}:
+        dhcp4: true
+        addresses:
+          - ${ipaddr}/24
+EOF_MAAS_CFG`,
+  ];
+
+  shellExec(`sudo chroot ${nfsHostPath} /usr/bin/qemu-aarch64-static /bin/bash <<'EOF'
+${steps
+  .map(
+    (s, i) => `echo "step ${i + 1}/${steps.length}: ${s.split('\n')[0]}"
+${s}
+`,
+  )
+  .join(``)}
+EOF`);
+
+  shellExec(`sudo chroot ${nfsHostPath} /usr/bin/qemu-aarch64-static /bin/bash <<'EOF'
+echo "nameserver ${process.env.MAAS_DNS}" | tee /etc/resolv.conf > /dev/null
+apt update
+EOF`);
+};
+
 try {
   switch (operator) {
     case 'save':
@@ -734,8 +816,8 @@ try {
       shellExec(`node bin/deploy update-dependencies`);
       shellExec(`auto-changelog`);
       shellExec(`node bin/build dd`);
-      shellExec(`node bin deploy dd --build-manifest --sync --info-router`);
-      shellExec(`node bin deploy dd production --build-manifest --sync --info-router`);
+      shellExec(`node bin deploy --kubeadm --build-manifest --sync --info-router --replicas 1 dd`);
+      shellExec(`node bin deploy --kubeadm --build-manifest --sync --info-router --replicas 1 dd production`);
       break;
     }
 
@@ -865,157 +947,125 @@ ${shellExec(`git log | grep Author: | sort -u`, { stdout: true }).split(`\n`).jo
 
       break;
     }
-    case 'ssh-export-server-keys': {
-      fs.copyFile('/etc/ssh/ssh_host_rsa_key', './engine-private/deploy/ssh_host_rsa_key');
-      fs.copyFile('/etc/ssh/ssh_host_rsa_key.pub', './engine-private/deploy/ssh_host_rsa_key.pub');
-      break;
-    }
-    case 'ssh-import-server-keys': {
-      fs.copyFile('./engine-private/deploy/ssh_host_rsa_key', '/etc/ssh/ssh_host_rsa_key');
-      fs.copyFile('./engine-private/deploy/ssh_host_rsa_key.pub', '/etc/ssh/ssh_host_rsa_key.pub');
-      break;
-    }
-    case 'ssh-import-client-keys': {
-      const host = process.argv[3];
-      shellExec(
-        `node bin/deploy set-ssh-keys ./engine-private/deploy/ssh_host_rsa_key ${host ? ` ${host}` : ``} ${
-          process.argv.includes('clean') ? 'clean' : ''
-        }`,
-      );
-      break;
-    }
-    case 'ssh-keys': {
-      // create ssh keys
-      const sshAccount = process.argv[3]; // [sudo username]@[host/ip]
-      const destPath = process.argv[4];
-      // shellExec(`ssh-keygen -t ed25519 -C "${sshAccount}" -f ${destPath}`);
-      if (fs.existsSync(destPath)) {
-        fs.removeSync(destPath);
-        fs.removeSync(destPath + '.pub');
-      }
-      shellExec(`ssh-keygen -t rsa -b 4096 -C "${sshAccount}" -f ${destPath}`);
-      // add host to keyscan
-      // shellExec(`ssh-keyscan -t rsa ${sshAccount.split(`@`)[1]} >> ~/.ssh/known_hosts`);
-      break;
-    }
-
-    case 'set-ssh-keys': {
-      const files = ['authorized_keys', 'id_rsa', 'id_rsa.pub', 'known_hosts ', 'known_hosts.old'];
-
-      // > write
-      // >> append
-
-      // /root/.ssh/id_rsa
-      // /root/.ssh/id_rsa.pub
-      if (process.argv.includes('clean')) {
-        for (const file of files) {
-          if (fs.existsSync(`/root/.ssh/${file}`)) {
-            logger.info('remove', `/root/.ssh/${file}`);
-            fs.removeSync(`/root/.ssh/${file}`);
-          }
-          fs.writeFileSync(`/root/.ssh/${file}`, '', 'utf8');
-        }
-        shellExec('eval `ssh-agent -s`' + ` && ssh-add -D`);
-      }
-
-      const destPath = process.argv[3];
-      const sshAuthKeyTarget = '/root/.ssh/authorized_keys';
-      if (!fs.existsSync(sshAuthKeyTarget)) shellExec(`touch ${sshAuthKeyTarget}`);
-      shellExec(`cat ${destPath}.pub > ${sshAuthKeyTarget}`);
-      shellExec(`cat ${destPath} >> ${sshAuthKeyTarget}`);
-
-      if (!fs.existsSync('/root/.ssh/id_rsa')) shellExec(`touch ${'/root/.ssh/id_rsa'}`);
-      shellExec(`cat ${destPath} > ${'/root/.ssh/id_rsa'}`);
-
-      if (!fs.existsSync('/root/.ssh/id_rsa.pub')) shellExec(`touch ${'/root/.ssh/id_rsa.pub'}`);
-      shellExec(`cat ${destPath}.pub > ${'/root/.ssh/id_rsa.pub'}`);
-
-      shellExec(`chmod 700 /root/.ssh/`);
-      for (const file of files) {
-        shellExec(`chmod 600 /root/.ssh/${file}`);
-      }
-      const host = process.argv[4];
-      // add key
-      shellExec('eval `ssh-agent -s`' + ' && ssh-add /root/.ssh/id_rsa' + ' && ssh-add -l');
-      if (host) shellExec(`ssh-keyscan -H ${host} >> ~/.ssh/known_hosts`);
-      shellExec(`sudo systemctl enable ssh`);
-      shellExec(`sudo systemctl restart ssh`);
-      shellExec(`sudo systemctl status ssh`);
-
-      break;
-    }
 
     case 'ssh': {
-      if (process.argv.includes('rocky')) {
-        shellExec(`sudo systemctl enable sshd`);
+      const host = process.argv[3] ?? `root@${await ip.public.ipv4()}`;
+      const domain = host.split('@')[1];
+      const user = 'root'; // host.split('@')[0];
+      const password = process.argv[4] ?? '';
+      const port = 22;
+
+      const setUpSSH = () => {
+        // Required port forwarding mapping
+        // ssh	TCP	2222	22	<local-server-ip>
+        // ssh	UDP	2222	22	<local-server-ip>
+
+        // Remote connect via public key
+        // ssh -i <key-path> <user>@<host>:2222
+
+        shellExec(`cat ./engine-private/deploy/id_rsa.pub > ~/.ssh/authorized_keys`);
+
+        // local trust on first use validator
+        // check ~/.ssh/known_hosts
+
+        // shellExec(`sudo sed -i -e "s@#PasswordAuthentication yes@PasswordAuthentication no@g" /etc/ssh/sshd_config`);
+        // shellExec(`sudo sed -i -e "s@#UsePAM no@UsePAM yes@g" /etc/ssh/sshd_config`);
+
+        // Include /etc/ssh/sshd_config.d/*.conf
+        // sudo tee /etc/ssh/sshd_config.d/99-custom.conf
+        shellExec(`sudo tee /etc/ssh/sshd_config <<EOF
+PasswordAuthentication	no
+ChallengeResponseAuthentication	yes
+UsePAM	yes
+PubkeyAuthentication	Yes
+RSAAuthentication	Yes
+PermitRootLogin	Yes
+X11Forwarding	yes
+X11DisplayOffset	10
+LoginGraceTime	120
+StrictModes	yes
+SyslogFacility	AUTH
+LogLevel	INFO
+#HostKey	/etc/ssh/ssh_host_ecdsa_key
+HostKey	/etc/ssh/ssh_host_ed25519_key
+#HostKey	/etc/ssh/ssh_host_rsa_key
+AuthorizedKeysFile	~/.ssh/authorized_keys
+Subsystem	sftp	/usr/libexec/openssh/sftp-server
+ListenAddress 0.0.0.0
+ListenAddress ::
+ListenAddress ${domain}
+ListenAddress ${domain}:22
+EOF`);
+
+        shellExec(`sudo chmod 700 ~/.ssh/`);
+        shellExec(`sudo chmod 600 ~/.ssh/authorized_keys`);
+        shellExec(`sudo chmod 644 ~/.ssh/known_hosts`);
+        shellExec(`sudo chmod 600 ~/.ssh/id_rsa`);
+        shellExec(`sudo chmod 600 /etc/ssh/ssh_host_ed25519_key`);
+        shellExec(`chown -R ${user}:${user} ~/.ssh`);
+
+        shellExec(`ufw allow ${port}/tcp`);
+        shellExec(`ufw allow ${port}/udp`);
+        shellExec(`ufw allow ssh`);
+        shellExec(`ufw allow from 192.168.0.0/16 to any port 22`);
+
+        // active ssh-agent
+        shellExec('eval `ssh-agent -s`' + ` && ssh-add ~/.ssh/id_rsa` + ` && ssh-add -l`);
+        // remove all
+        // shellExec(`ssh-add -D`);
+        // remove single
+        // shellExec(`ssh-add -d ~/.ssh/id_rsa`);
+
+        // shellExec(`echo "@${host.split(`@`)[1]} * $(cat ~/.ssh/id_rsa.pub)" > ~/.ssh/known_hosts`);
+        shellExec('eval `ssh-agent -s`' + `&& ssh-keyscan -H -t ed25519 ${host.split(`@`)[1]} > ~/.ssh/known_hosts`);
+        // shellExec(`sudo echo "" > ~/.ssh/known_hosts`);
+
+        // ssh-copy-id -i ~/.ssh/id_rsa.pub -p <port_number> <username>@<host>
+        shellExec(`ssh-copy-id -i ~/.ssh/id_rsa.pub -p ${port} ${host}`);
+        // debug:
+        // shellExec(`ssh -vvv ${host}`);
+
+        shellExec(`sudo cp ./engine-private/deploy/id_rsa ~/.ssh/id_rsa`);
+        shellExec(`sudo cp ./engine-private/deploy/id_rsa.pub ~/.ssh/id_rsa.pub`);
+
+        shellExec(`sudo echo "" > /etc/ssh/ssh_host_ecdsa_key`);
+        shellExec(`sudo cp ./engine-private/deploy/id_rsa /etc/ssh/ssh_host_ed25519_key`);
+        shellExec(`sudo echo "" > /etc/ssh/ssh_host_rsa_key`);
+
+        shellExec(`sudo echo "" > /etc/ssh/ssh_host_ecdsa_key.pub`);
+        shellExec(`sudo cp ./engine-private/deploy/id_rsa.pub /etc/ssh/ssh_host_ed25519_key.pub`);
+        shellExec(`sudo echo "" > /etc/ssh/ssh_host_rsa_key.pub`);
 
-        shellExec(`sudo systemctl start sshd`);
+        shellExec(`sudo systemctl enable sshd`);
+        shellExec(`sudo systemctl restart sshd`);
 
-        shellExec(`sudo systemctl status sshd`);
+        const status = shellExec(`sudo systemctl status sshd`, { silent: true, stdout: true });
+        console.log(
+          status.match('running') ? status.replaceAll(`running`, `running`.green) : `ssh service not running`.red,
+        );
+      };
 
-        shellExec(`sudo ss -lt`);
-      } else {
-        if (!process.argv.includes('server')) {
-          shellExec(`sudo apt update`);
-          shellExec(`sudo apt install openssh-server -y`);
-          shellExec(`sudo apt install ssh-askpass`);
-        }
-        shellExec(`sudo systemctl enable ssh`);
-        shellExec(`sudo systemctl restart ssh`);
-        shellExec(`sudo systemctl status ssh`);
+      if (process.argv.includes('import')) {
+        setUpSSH();
+        break;
       }
-      // sudo service ssh restart
-      shellExec(`ip a`);
-
-      // adduser newuser
-      // usermod -aG sudo newuser
-
-      // ssh -i '/path/to/keyfile' username@server
 
-      // ssh-keygen -t ed25519 -C "your_email@example.com" -f $HOME/.ssh/id_rsa
+      shellExec(`sudo rm -rf ./id_rsa`);
+      shellExec(`sudo rm -rf ./id_rsa.pub`);
 
-      // legacy: ssh-keygen -t rsa -b 4096 -C "your_email@example.com" -f $HOME/.ssh/id_rsa
+      if (process.argv.includes('legacy'))
+        shellExec(`ssh-keygen -t rsa -b 4096 -f id_rsa -N "${password}" -q -C "${host}"`);
+      else shellExec(`ssh-keygen -t ed25519 -f id_rsa -N "${password}" -q -C "${host}"`);
 
-      // vi .ssh/authorized_keys
-      // chmod 700 .ssh
-      // chmod 600 authorized_keys
+      shellExec(`sudo cp ./id_rsa ~/.ssh/id_rsa`);
+      shellExec(`sudo cp ./id_rsa.pub ~/.ssh/id_rsa.pub`);
 
-      // cat id_rsa.pub > .ssh/authorized_keys
-
-      // add public key to authorized keys
-      // cat .ssh/id_ed25519.pub | ssh [sudo username]@[host/ip] 'cat >> .ssh/authorized_keys'
-
-      // 2. Open /etc/ssh/sshd_config file
-      // nano /etc/ssh/sshd_config
-
-      // 3. add example code to last line of file
-      // Match User newuser
-      //   PasswordAuthentication yes
-
-      // ssh [sudo username]@[host/ip]
-      // open port 22
-
-      // init ssh agent service
-      // eval `ssh-agent -s`
-
-      // list keys
-      // ssh-add -l
-
-      // add key
-      // ssh-add /root/.ssh/id_rsa
-
-      // remove
-      // ssh-add -d /path/to/private/key
-
-      // remove all
-      // ssh-add -D
-
-      // sshpass -p ${{ secrets.PSWD }} ssh -o StrictHostKeyChecking=no -p 22 ${{ secrets.USER}}@${{ secrets.VPS_IP }} 'cd /home/adam && ./deploy.sh'
-
-      // copies the public key of your default identity (use -i identity_file for other identities) to the remote host.
-      // ssh-copy-id user@hostname.example.com
-      // ssh-copy-id "user@hostname.example.com -p <port-number>"
+      shellExec(`sudo cp ./id_rsa ./engine-private/deploy/id_rsa`);
+      shellExec(`sudo cp ./id_rsa.pub ./engine-private/deploy/id_rsa.pub`);
 
+      shellExec(`sudo rm -rf ./id_rsa`);
+      shellExec(`sudo rm -rf ./id_rsa.pub`);
+      setUpSSH();
       break;
     }
 
@@ -1092,8 +1142,1269 @@ ${shellExec(`git log | grep Author: | sort -u`, { stdout: true }).split(`\n`).jo
       break;
     }
 
-    default:
+    case 'monitor': {
+      shellExec(
+        `node bin monitor ${process.argv[6] === 'sync' ? '--sync ' : ''}--type ${process.argv[3]} ${process.argv[4]} ${
+          process.argv[5]
+        }`,
+        {
+          async: true,
+        },
+      );
+      break;
+    }
+
+    case 'postgresql': {
+      if (process.argv.includes('install')) {
+        shellExec(`sudo dnf install -y postgresql-server postgresql`);
+        shellExec(`sudo postgresql-setup --initdb`);
+        shellExec(`chown postgres /var/lib/pgsql/data`);
+        shellExec(`sudo systemctl enable postgresql.service`);
+        shellExec(`sudo systemctl start postgresql.service`);
+      } else {
+        shellExec(`sudo systemctl enable postgresql.service`);
+        shellExec(`sudo systemctl restart postgresql.service`);
+      }
+
+      shellExec(`sudo systemctl status postgresql.service`);
+
+      // sudo systemctl stop postgresql
+      // sudo systemctl disable postgresql
+
+      // psql login
+      // psql -U <user> -h 127.0.0.1 -W <db-name>
+
+      // gedit /var/lib/pgsql/data/pg_hba.conf
+      // host    <db-name>    	<db-user>        <db-host>               md5
+      // local   all             postgres                                trust
+      // # "local" is for Unix domain socket connections only
+      // local   all             all                                     md5
+      // # IPv4 local connections:
+      // host    all             all             127.0.0.1/32            md5
+      // # IPv6 local connections:
+      // host    all             all             ::1/128                 md5
+
+      // gedit /var/lib/pgsql/data/postgresql.conf
+      // listen_addresses = '*'
+
+      break;
+    }
+
+    case 'postgresql-14': {
+      shellExec(`sudo /usr/pgsql-14/bin/postgresql-14-setup initdb`);
+      shellExec(`sudo systemctl start postgresql-14`);
+      shellExec(`sudo systemctl enable postgresql-14`);
+      shellExec(`sudo systemctl status postgresql-14`);
+      // sudo dnf install postgresql14-contrib
+      break;
+    }
+
+    case 'pg-stop': {
+      shellExec(`sudo systemctl stop postgresql-14`);
+      shellExec(`sudo systemctl disable postgresql-14`);
+      break;
+    }
+    case 'pg-start': {
+      shellExec(`sudo systemctl enable postgresql-14`);
+      shellExec(`sudo systemctl restart postgresql-14`);
+      break;
+    }
+
+    case 'pg-list-db': {
+      shellExec(`sudo -i -u postgres psql -c "\\l"`);
+      break;
+    }
+
+    case 'pg-list-table': {
+      shellExec(`sudo -i -u postgres psql -c "\\dt *.*"`);
+      // schema_name.*
       break;
+    }
+    case 'pg-drop-db': {
+      shellExec(`sudo -i -u postgres psql -c "DROP DATABASE ${process.argv[3]} WITH (FORCE)"`);
+      shellExec(`sudo -i -u postgres psql -c "DROP USER ${process.argv[4]}"`);
+      break;
+    }
+
+    case 'maas-stop': {
+      shellExec(`sudo snap stop maas`);
+      break;
+    }
+
+    case 'maas': {
+      dotenv.config({ path: `${getUnderpostRootPath()}/.env`, override: true });
+      const IP_ADDRESS = getLocalIPv4Address();
+      const serverip = IP_ADDRESS;
+      const tftpRoot = process.env.TFTP_ROOT;
+      const ipaddr = process.env.RPI4_IP;
+      const netmask = process.env.NETMASK;
+      const gatewayip = process.env.GATEWAY_IP;
+
+      let resources;
+      try {
+        resources = JSON.parse(
+          shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} boot-resources read`, {
+            silent: true,
+            stdout: true,
+          }),
+        ).map((o) => ({
+          id: o.id,
+          name: o.name,
+          architecture: o.architecture,
+        }));
+      } catch (error) {
+        logger.error(error);
+      }
+
+      const machineFactory = (m) => ({
+        system_id: m.interface_set[0].system_id,
+        mac_address: m.interface_set[0].mac_address,
+        hostname: m.hostname,
+        status_name: m.status_name,
+      });
+
+      let machines;
+      try {
+        machines = JSON.parse(
+          shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} machines read`, {
+            stdout: true,
+            silent: true,
+          }),
+        ).map((m) => machineFactory(m));
+      } catch (error) {
+        logger.error(error);
+      }
+
+      if (process.argv.includes('db')) {
+        // DROP, ALTER, CREATE, WITH ENCRYPTED
+        // sudo -u <user> -h <host> psql <db-name>
+        shellExec(`DB_PG_MAAS_NAME=${process.env.DB_PG_MAAS_NAME}`);
+        shellExec(`DB_PG_MAAS_PASS=${process.env.DB_PG_MAAS_PASS}`);
+        shellExec(`DB_PG_MAAS_USER=${process.env.DB_PG_MAAS_USER}`);
+        shellExec(`DB_PG_MAAS_HOST=${process.env.DB_PG_MAAS_HOST}`);
+        shellExec(
+          `sudo -i -u postgres psql -c "CREATE USER \"$DB_PG_MAAS_USER\" WITH ENCRYPTED PASSWORD '$DB_PG_MAAS_PASS'"`,
+        );
+        shellExec(
+          `sudo -i -u postgres psql -c "ALTER USER \"$DB_PG_MAAS_USER\" WITH ENCRYPTED PASSWORD '$DB_PG_MAAS_PASS'"`,
+        );
+        const actions = ['LOGIN', 'SUPERUSER', 'INHERIT', 'CREATEDB', 'CREATEROLE', 'REPLICATION'];
+        shellExec(`sudo -i -u postgres psql -c "ALTER USER \"$DB_PG_MAAS_USER\" WITH ${actions.join(' ')}"`);
+        shellExec(`sudo -i -u postgres psql -c "\\du"`);
+
+        shellExec(`sudo -i -u postgres createdb -O "$DB_PG_MAAS_USER" "$DB_PG_MAAS_NAME"`);
+
+        shellExec(`sudo -i -u postgres psql -c "\\l"`);
+      }
+
+      if (process.argv.includes('ls')) {
+        shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} boot-sources read`);
+        shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} commissioning-scripts read`);
+        // shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} boot-source-selections read 60`);
+        console.table(resources);
+        console.table(machines);
+        process.exit(0);
+      }
+
+      // TODO: - Disable maas proxy (egress forwarding to public dns)
+      //       - Configure maas dns forwarding ${process.env.MAAS_DNS}
+      //       - Enable DNSSEC validation of upstream zones: Automatic (use default root key)
+
+      if (process.argv.includes('clear')) {
+        for (const machine of machines) {
+          shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} machine delete ${machine.system_id}`);
+        }
+        // machines = [];
+        shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} discoveries clear all=true`);
+        if (process.argv.includes('force')) {
+          shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} discoveries scan force=true`);
+        }
+        process.exit(0);
+      }
+      if (process.argv.includes('grub-arm64')) {
+        shellExec(`sudo dnf install grub2-efi-aa64-modules`);
+        shellExec(`sudo dnf install grub2-efi-x64-modules`);
+        // sudo grub2-mknetdir --net-directory=${tftpRoot} --subdir=/boot/grub --module-path=/usr/lib/grub/arm64-efi arm64-efi
+        process.exit(0);
+      }
+
+      if (process.argv.includes('psql')) {
+        const cmd = `psql -U ${process.env.DB_PG_MAAS_USER} -h ${process.env.DB_PG_MAAS_HOST} -W ${process.env.DB_PG_MAAS_NAME}`;
+        pbcopy(cmd);
+        process.exit(0);
+      }
+      if (process.argv.includes('logs')) {
+        shellExec(`maas status`);
+        const cmd = `journalctl -f -t dhcpd -u snap.maas.pebble.service`;
+        pbcopy(cmd);
+        process.exit(0);
+      }
+      if (process.argv.includes('reset')) {
+        // shellExec(
+        //   `maas init region+rack --database-uri "postgres://$DB_PG_MAAS_USER:$DB_PG_MAAS_PASS@$DB_PG_MAAS_HOST/$DB_PG_MAAS_NAME"` +
+        //     ` --maas-url http://${IP_ADDRESS}:5240/MAAS`,
+        // );
+        const cmd =
+          `maas init region+rack --database-uri "postgres://${process.env.DB_PG_MAAS_USER}:${process.env.DB_PG_MAAS_PASS}@${process.env.DB_PG_MAAS_HOST}/${process.env.DB_PG_MAAS_NAME}"` +
+          ` --maas-url http://${IP_ADDRESS}:5240/MAAS`;
+        pbcopy(cmd);
+        process.exit(0);
+      }
+      if (process.argv.includes('dhcp')) {
+        const snippets = JSON.parse(
+          shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} dhcpsnippets read`, {
+            stdout: true,
+            silent: true,
+            disableLog: true,
+          }),
+        );
+        for (const snippet of snippets) {
+          switch (snippet.name) {
+            case 'arm64':
+              snippet.value = snippet.value.split(`\n`);
+              snippet.value[1] = `          filename "http://${IP_ADDRESS}:5248/images/bootloaders/uefi/arm64/grubaa64.efi";`;
+              snippet.value[5] = `          filename "http://${IP_ADDRESS}:5248/images/bootloaders/uefi/arm64/grubaa64.efi";`;
+              snippet.value = snippet.value.join(`\n`);
+              shellExec(
+                `maas ${process.env.MAAS_ADMIN_USERNAME} dhcpsnippet update ${snippet.name} value='${snippet.value}'`,
+              );
+              break;
+
+            default:
+              break;
+          }
+        }
+
+        console.log(snippets);
+
+        process.exit(0);
+      }
+      // shellExec(`MAAS_ADMIN_USERNAME=${process.env.MAAS_ADMIN_USERNAME}`);
+      // shellExec(`MAAS_ADMIN_EMAIL=${process.env.MAAS_ADMIN_EMAIL}`);
+      // shellExec(`maas createadmin --username $MAAS_ADMIN_USERNAME --email $MAAS_ADMIN_EMAIL`);
+
+      // MaaS admin CLI:
+      // maas login <maas-admin-username> http://localhost:5240/MAAS
+      // paste GUI API KEY (profile section)
+
+      // Import custom image
+      // maas <maas-admin-username> boot-resources create name='custom/RockyLinuxRpi4' \
+      // title='RockyLinuxRpi4' \
+      // architecture='arm64/generic' \
+      // filetype='tgz' \
+      // content@=/home/RockyLinuxRpi_9-latest.tar.gz
+
+      // Image boot resource:
+      // /var/snap/maas/current/root/snap/maas
+      // /var/snap/maas/common/maas/tftp_root
+      // sudo chmod 755 /var/snap/maas/common/maas/tftp_root
+
+      // /var/snap/maas/common/maas/dhcpd.conf
+      // sudo snap restart maas.pebble
+
+      // PXE Linux files:
+      // /var/snap/maas/common/maas/image-storage/bootloaders/pxe/i386
+      // sudo nmcli con modify <interface-device-name-connection-id> ethtool.feature-rx on ethtool.feature-tx off
+      // sudo nmcli connection up <interface-device-name-connection-id>
+
+      // man nm-settings |grep feature-tx-checksum
+
+      // nmcli c modify <interface-device-name-connection-id> \
+      //  ethtool.feature-tx-checksum-fcoe-crc off \
+      //  ethtool.feature-tx-checksum-ip-generic off \
+      //  ethtool.feature-tx-checksum-ipv4 off \
+      //  ethtool.feature-tx-checksum-ipv6 off \
+      //  ethtool.feature-tx-checksum-sctp off
+
+      // Ensure Rocky NFS server and /etc/exports configured
+      // sudo systemctl restart nfs-server
+      // Check mounts: showmount -e <server-ip>
+      // Check nfs ports: rpcinfo -p
+      // sudo chown -R root:root ${process.env.NFS_EXPORT_PATH}/rpi4mb
+      // sudo chmod 755 ${process.env.NFS_EXPORT_PATH}/rpi4mb
+
+      // tftp server
+      // sudo chown -R root:root /var/snap/maas/common/maas/tftp_root/rpi4mb
+
+      // tftp client
+      // sudo dnf install tftp
+      // tftp <server-ip> -c get <path>
+
+      // Check firewall-cmd
+      // firewall-cmd --permanent --add-service=rpc-bind
+      // firewall-cmd --reload
+      // systemctl disable firewalld
+      // sudo firewall-cmd --permanent --add-port=10259/tcp --zone=public
+
+      // Image extension transform (.img.xz to .tar.gz):
+      // tar -cvzf image-name.tar.gz image-name.img.xz
+
+      // Rocky network configuration:
+      // /etc/NetworkManager/system-connections
+
+      // Rocky kernel params update
+      // sudo grubby --args="<key>=<value> <key>=<value>" --update-kernel=ALL
+      // sudo reboot now
+
+      // Temporal:
+      // sudo snap install temporal
+      // journalctl -u snap.maas.pebble -t maas-regiond
+      // journalctl -u snap.maas.pebble -t maas-temporal -n 100 --no-pager -f
+
+      // Remove:
+      // sudo dnf remove <package> -y; sudo dnf autoremove -y; sudo dnf clean packages
+      // check: ~
+      // check: ~./cache
+      // check: ~./config
+
+      // Check file logs
+      // grep -i -E -C 1 '<key-a>|<key-b>' /example.log | tail -n 600
+
+      // Back into your firmware setup (UEFI or BIOS config screen).
+      // grub> fwsetup
+
+      // Poweroff:
+      // grub > halt
+      // initramfs > poweroff
+
+      // Check interface
+      // ip link show
+      // nmcli con show
+
+      let firmwarePath,
+        tftpSubDir,
+        kernelFilesPaths,
+        name,
+        architecture,
+        resource,
+        nfsConnectStr,
+        etcExports,
+        nfsServerRootPath,
+        bootConf,
+        zipFirmwareFileName,
+        zipFirmwareName,
+        zipFirmwareUrl,
+        interfaceName,
+        nfsHost;
+
+      switch (process.argv[3]) {
+        case 'rpi4mb':
+          const resourceId = process.argv[4] ?? '39';
+          tftpSubDir = '/rpi4mb';
+          zipFirmwareFileName = `RPi4_UEFI_Firmware_v1.41.zip`;
+          zipFirmwareName = zipFirmwareFileName.split('.zip')[0];
+          zipFirmwareUrl = `https://github.com/pftf/RPi4/releases/download/v1.41/RPi4_UEFI_Firmware_v1.41.zip`;
+          firmwarePath = `../${zipFirmwareName}`;
+          interfaceName = process.env.RPI4_INTERFACE_NAME;
+          nfsHost = 'rpi4mb';
+          if (!fs.existsSync(firmwarePath)) {
+            await Downloader(zipFirmwareUrl, `../${zipFirmwareFileName}`);
+            shellExec(`cd .. && mkdir ${zipFirmwareName} && cd ${zipFirmwareName} && unzip ../${zipFirmwareFileName}`);
+          }
+          resource = resources.find((o) => o.id == resourceId);
+          name = resource.name;
+          architecture = resource.architecture;
+          resource = resources.find((o) => o.name === name && o.architecture === architecture);
+          nfsServerRootPath = `${process.env.NFS_EXPORT_PATH}/rpi4mb`;
+          // ,anonuid=1001,anongid=100
+          // etcExports = `${nfsServerRootPath} *(rw,all_squash,sync,no_root_squash,insecure)`;
+          etcExports = `${nfsServerRootPath} 192.168.1.0/24(${[
+            'rw',
+            // 'all_squash',
+            'sync',
+            'no_root_squash',
+            'no_subtree_check',
+            'insecure',
+          ]})`;
+          const resourceData = JSON.parse(
+            shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} boot-resource read ${resource.id}`, {
+              stdout: true,
+              silent: true,
+              disableLog: true,
+            }),
+          );
+          const bootFiles = resourceData.sets[Object.keys(resourceData.sets)[0]].files;
+          const suffix = architecture.match('xgene') ? '.xgene' : '';
+
+          kernelFilesPaths = {
+            'vmlinuz-efi': bootFiles['boot-kernel' + suffix].filename_on_disk,
+            'initrd.img': bootFiles['boot-initrd' + suffix].filename_on_disk,
+            squashfs: bootFiles['squashfs'].filename_on_disk,
+          };
+          const protocol = 'tcp'; // v3 -> tcp, v4 -> udp
+
+          const mountOptions = [
+            protocol,
+            'vers=3',
+            'nfsvers=3',
+            'nolock',
+            // 'protocol=tcp',
+            // 'hard=true',
+            'port=2049',
+            // 'sec=none',
+            'rw',
+            'hard',
+            'intr',
+            'rsize=32768',
+            'wsize=32768',
+            'acregmin=0',
+            'acregmax=0',
+            'acdirmin=0',
+            'acdirmax=0',
+            'noac',
+            // 'nodev',
+            // 'nosuid',
+          ];
+          const cmd = [
+            `console=serial0,115200`,
+            `console=tty1`,
+            // `initrd=-1`,
+            // `net.ifnames=0`,
+            // `dwc_otg.lpm_enable=0`,
+            // `elevator=deadline`,
+            `root=/dev/nfs`,
+            `nfsroot=${serverip}:${process.env.NFS_EXPORT_PATH}/rpi4mb,${mountOptions}`,
+            // `nfsroot=${serverip}:${process.env.NFS_EXPORT_PATH}/rpi4mb`,
+            `ip=${ipaddr}:${serverip}:${gatewayip}:${netmask}:${nfsHost}:${interfaceName}:static`,
+            `rootfstype=nfs`,
+            `rw`,
+            `rootwait`,
+            `fixrtc`,
+            'initrd=initrd.img',
+            // 'boot=casper',
+            // 'ro',
+            'netboot=nfs',
+            `cloud-config-url=/dev/null`,
+            // 'ip=dhcp',
+            // 'ip=dfcp',
+            // 'autoinstall',
+            // 'rd.break',
+          ];
+
+          nfsConnectStr = cmd.join(' ');
+          bootConf = `[all]
+MAC_ADDRESS=00:00:00:00:00:00
+MAC_ADDRESS_OTP=0,1
+BOOT_UART=0
+WAKE_ON_GPIO=1
+POWER_OFF_ON_HALT=0
+ENABLE_SELF_UPDATE=1
+DISABLE_HDMI=0
+TFTP_IP=${serverip}
+TFTP_PREFIX=1
+TFTP_PREFIX_STR=${tftpSubDir.slice(1)}/
+NET_INSTALL_ENABLED=1
+DHCP_TIMEOUT=45000
+DHCP_REQ_TIMEOUT=4000
+TFTP_FILE_TIMEOUT=30000
+BOOT_ORDER=0x21`;
+
+          break;
+
+        default:
+          break;
+      }
+      shellExec(`sudo chmod 755 ${process.env.NFS_EXPORT_PATH}/${nfsHost}`);
+
+      shellExec(`sudo rm -rf ${tftpRoot}${tftpSubDir}`);
+      shellExec(`sudo cp -a ${firmwarePath} ${tftpRoot}${tftpSubDir}`);
+      shellExec(`mkdir -p ${tftpRoot}${tftpSubDir}/pxe`);
+
+      fs.writeFileSync(`/etc/exports`, etcExports, 'utf8');
+      if (bootConf) fs.writeFileSync(`${tftpRoot}${tftpSubDir}/boot.conf`, bootConf, 'utf8');
+
+      shellExec(`node bin/deploy nfs`);
+
+      if (process.argv.includes('restart')) {
+        shellExec(`sudo snap restart maas.pebble`);
+        let secs = 0;
+        while (
+          !(
+            shellExec(`maas status`, { silent: true, disableLog: true, stdout: true })
+              .split(' ')
+              .filter((l) => l.match('inactive')).length === 1
+          )
+        ) {
+          await timer(1000);
+          console.log(`Waiting... (${++secs}s)`);
+        }
+      }
+
+      switch (process.argv[3]) {
+        case 'rpi4mb':
+          {
+            // subnet DHCP snippets
+            // # UEFI ARM64
+            // if option arch = 00:0B {
+            //   filename "rpi4mb/pxe/grubaa64.efi";
+            // }
+            // elsif option arch = 00:13 {
+            //   filename "http://<IP_ADDRESS>:5248/images/bootloaders/uefi/arm64/grubaa64.efi";
+            //   option vendor-class-identifier "HTTPClient";
+            // }
+            for (const file of ['bootaa64.efi', 'grubaa64.efi']) {
+              shellExec(
+                `sudo cp -a /var/snap/maas/common/maas/image-storage/bootloaders/uefi/arm64/${file} ${tftpRoot}${tftpSubDir}/pxe/${file}`,
+              );
+            }
+            // const file = 'bcm2711-rpi-4-b.dtb';
+            // shellExec(
+            //   `sudo cp -a  ${firmwarePath}/${file} /var/snap/maas/common/maas/image-storage/bootloaders/uefi/arm64/${file}`,
+            // );
+
+            // const ipxeSrc = fs
+            //   .readFileSync(`${tftpRoot}/ipxe.cfg`, 'utf8')
+            //   .replaceAll('amd64', 'arm64')
+            //   .replaceAll('${next-server}', IP_ADDRESS);
+            // fs.writeFileSync(`${tftpRoot}/ipxe.cfg`, ipxeSrc, 'utf8');
+
+            {
+              for (const file of Object.keys(kernelFilesPaths)) {
+                shellExec(
+                  `sudo cp -a /var/snap/maas/common/maas/image-storage/${kernelFilesPaths[file]} ${tftpRoot}${tftpSubDir}/pxe/${file}`,
+                );
+              }
+              // const configTxtSrc = fs.readFileSync(`${firmwarePath}/config.txt`, 'utf8');
+              // fs.writeFileSync(
+              //   `${tftpRoot}${tftpSubDir}/config.txt`,
+              //   configTxtSrc
+              //     .replace(`kernel=kernel8.img`, `kernel=vmlinuz`)
+              //     .replace(`# max_framebuffers=2`, `max_framebuffers=2`)
+              //     .replace(`initramfs initramfs8 followkernel`, `initramfs initrd.img followkernel`),
+              //   'utf8',
+              // );
+
+              // grub:
+              // set root=(pxe)
+
+              // UNDERPOST.NET UEFI/GRUB/MAAS RPi4 commissioning (ARM64)
+              const menuentryStr = 'underpost.net rpi4mb maas commissioning (ARM64)';
+              const grubCfgPath = `${tftpRoot}/grub/grub.cfg`;
+              fs.writeFileSync(
+                grubCfgPath,
+                `
+    insmod gzio
+    insmod http
+    insmod nfs
+    set timeout=5
+    set default=0
+    
+    menuentry '${menuentryStr}' {
+      set root=(tftp,${serverip}) 
+      linux ${tftpSubDir}/pxe/vmlinuz-efi ${nfsConnectStr}
+      initrd ${tftpSubDir}/pxe/initrd.img
+      boot
+    }
+    
+        `,
+                'utf8',
+              );
+            }
+            const arm64EfiPath = `${tftpRoot}/grub/arm64-efi`;
+            if (fs.existsSync(arm64EfiPath)) shellExec(`sudo rm -rf ${arm64EfiPath}`);
+            shellExec(`sudo cp -a /usr/lib/grub/arm64-efi ${arm64EfiPath}`);
+          }
+
+          break;
+
+        default:
+          break;
+      }
+
+      logger.info('succes maas deploy', {
+        resource,
+        kernelFilesPaths,
+        tftpRoot,
+        tftpSubDir,
+        firmwarePath,
+        etcExports,
+        nfsServerRootPath,
+        nfsConnectStr,
+      });
+      if (process.argv.includes('restart')) {
+        if (fs.existsSync(`node engine-private/r.js`)) shellExec(`node engine-private/r`);
+        shellExec(`node bin/deploy maas dhcp`);
+        shellExec(`sudo chown -R root:root ${tftpRoot}`);
+        shellExec(`sudo sudo chmod 755 ${tftpRoot}`);
+      }
+      // for (const machine of machines) {
+      //   // shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} machine delete ${machine.system_id}`);
+      //   shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} machine commission ${machine.system_id}`, {
+      //     silent: true,
+      //   });
+      // }
+      // machines = [];
+
+      const monitor = async () => {
+        // discoveries         Query observed discoveries.
+        // discovery           Read or delete an observed discovery.
+
+        const discoveries = JSON.parse(
+          shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} discoveries read`, {
+            silent: true,
+            stdout: true,
+          }),
+        ).filter(
+          (o) => o.ip !== IP_ADDRESS && o.ip !== gatewayip && !machines.find((_o) => _o.mac_address === o.mac_address),
+        );
+
+        //   {
+        //     "discovery_id": "",
+        //     "ip": "192.168.1.189",
+        //     "mac_address": "00:00:00:00:00:00",
+        //     "last_seen": "2025-05-05T14:17:37.354",
+        //     "hostname": null,
+        //     "fabric_name": "",
+        //     "vid": null,
+        //     "mac_organization": "",
+        //     "observer": {
+        //         "system_id": "",
+        //         "hostname": "",
+        //         "interface_id": 1,
+        //         "interface_name": ""
+        //     },
+        //     "resource_uri": "/MAAS/api/2.0/discovery/MTkyLjE2OC4xLjE4OSwwMDowMDowMDowMDowMDowMA==/"
+        // },
+
+        for (const discovery of discoveries) {
+          const machine = {
+            architecture: architecture.match('amd') ? 'amd64/generic' : 'arm64/generic',
+            mac_address: discovery.mac_address,
+            hostname: discovery.hostname ?? discovery.mac_organization ?? discovery.domain ?? `generic-host-${s4()}`,
+            // discovery.ip.match(ipaddr)
+            //   ? nfsHost
+            //   : `unknown-${s4()}`,
+            // description: '',
+            // https://maas.io/docs/reference-power-drivers
+            power_type: 'manual', // manual
+            // power_parameters_power_address: discovery.ip,
+            mac_addresses: discovery.mac_address,
+          };
+          machine.hostname = machine.hostname.replaceAll(' ', '').replaceAll('.', '');
+
+          try {
+            let newMachine = shellExec(
+              `maas ${process.env.MAAS_ADMIN_USERNAME} machines create ${Object.keys(machine)
+                .map((k) => `${k}="${machine[k]}"`)
+                .join(' ')}`,
+              {
+                silent: true,
+                stdout: true,
+              },
+            );
+            newMachine = machineFactory(JSON.parse(newMachine));
+            machines.push(newMachine);
+            console.log(newMachine);
+            shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} machine commission ${newMachine.system_id}`, {
+              silent: true,
+            });
+          } catch (error) {
+            logger.error(error, error.stack);
+          }
+        }
+        // if (discoveries.length > 0) {
+        //   shellExec(
+        //     `maas ${process.env.MAAS_ADMIN_USERNAME} machines read | jq '.[] | {system_id: .interface_set[0].system_id, hostname, status_name, mac_address: .interface_set[0].mac_address}'`,
+        //   );
+        // }
+        await timer(1000);
+        monitor();
+      };
+      // shellExec(`node bin/deploy open-virtual-root ${architecture.match('amd') ? 'amd64' : 'arm64'} ${nfsHost}`);
+      machines = [];
+      shellExec(`node bin/deploy maas clear`);
+      monitor();
+      break;
+    }
+
+    case 'nfs': {
+      // Daemon RPC  NFSv3. ports:
+
+      // 2049 (TCP/UDP) – nfsd standard port.
+      // 111 (TCP/UDP) – rpcbind/portmapper.
+      // 20048 (TCP/UDP) – rpc.mountd.
+      // 32765 (TCP/UDP) – rpc.statd.
+      // 32766 (TCP/UDP) – lockd (NLM).
+
+      // Configure export and permissions:
+      // /etc/exports
+
+      // Configure ports:
+      // /etc/nfs.conf
+
+      fs.writeFileSync(
+        `/etc/nfs.conf`,
+        `
+[mountd]
+port = 20048
+
+[statd]
+port = 32765
+outgoing-port = 32765
+
+[nfsd]
+rdma=y
+rdma-port=20049
+
+[lockd]
+port = 32766
+udp-port = 32766
+        `,
+        'utf8',
+      );
+
+      // Client users have read-only access to resources and are identified as anonymous on the server.
+      // /share ip-client(ro,all_squash)
+
+      // Client users can modify resources and keep their UID on the server. Only root is identified as anonymous.
+      // /share ip-client(rw)
+
+      // Users on client workstation 1 can modify resources, while those on client workstation 2 have read-only access.
+      // UIDs are kept on the server, and only root is identified as anonymous.
+      // /share ip-client1(rw) ip-client2(ro)
+
+      // Client1 users can modify resources. Their UID is changed to 1001 and their GID to 100 on the server.
+      // /share ip-client(rw,all_squash,anonuid=1001,anongid=100)
+
+      // sudo dnf install nfs-utils
+      // sudo systemctl enable --now rpcbind    // RPC map service
+      // sudo systemctl enable --now nfs-server // nfs domains nfsd
+
+      // Update exports:
+      // shellExec(`sudo exportfs -a -r`);
+      // shellExec(`sudo exportfs -v`);
+
+      // Active nfs
+      shellExec(`sudo exportfs -s`);
+
+      shellExec(`sudo exportfs -rav`);
+
+      // Rocky enable virt_use_nfs
+      // sudo setsebool -P virt_use_nfs 1
+
+      // Disable share:
+      // sudo exportfs -u <client-ip>:${process.env.NFS_EXPORT_PATH}/rpi4mb
+
+      // Nfs client:
+      // mount -t nfs <server-ip>:/server-mnt /mnt
+      // umount /mnt
+
+      shellExec(`sudo systemctl restart nfs-server`);
+      break;
+    }
+    case 'update-virtual-root': {
+      dotenv.config({ path: `${getUnderpostRootPath()}/.env`, override: true });
+      const IP_ADDRESS = getLocalIPv4Address();
+      const architecture = process.argv[3];
+      const host = process.argv[4];
+      const nfsHostPath = `${process.env.NFS_EXPORT_PATH}/${host}`;
+      const ipaddr = process.env.RPI4_IP;
+      await updateVirtualRoot({
+        IP_ADDRESS,
+        architecture,
+        host,
+        nfsHostPath,
+        ipaddr,
+      });
+      break;
+    }
+    case 'open-virtual-root': {
+      dotenv.config({ path: `${getUnderpostRootPath()}/.env`, override: true });
+      const IP_ADDRESS = getLocalIPv4Address();
+      const architecture = process.argv[3];
+      const host = process.argv[4];
+      const nfsHostPath = `${process.env.NFS_EXPORT_PATH}/${host}`;
+      shellExec(`sudo dnf install -y iptables-legacy`);
+      shellExec(`sudo dnf install -y debootstrap`);
+      shellExec(`sudo dnf install kernel-modules-extra-$(uname -r)`);
+      switch (architecture) {
+        case 'arm64':
+          shellExec(`sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes`);
+
+          break;
+
+        default:
+          break;
+      }
+
+      shellExec(`sudo modprobe binfmt_misc`);
+      shellExec(`sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc`);
+
+      if (process.argv.includes('build')) {
+        // shellExec(`depmod -a`);
+        shellExec(`mkdir -p ${nfsHostPath}`);
+        let cmd;
+        switch (host) {
+          case 'rpi4mb':
+            shellExec(`sudo rm -rf ${nfsHostPath}/*`);
+            shellExec(`sudo chown -R root:root ${nfsHostPath}`);
+            cmd = [
+              `sudo debootstrap`,
+              `--arch=arm64`,
+              `--variant=minbase`,
+              `--foreign`, // arm64 on amd64
+              `noble`,
+              nfsHostPath,
+              `http://ports.ubuntu.com/ubuntu-ports/`,
+            ];
+            break;
+
+          default:
+            break;
+        }
+        shellExec(cmd.join(' '));
+
+        shellExec(`sudo podman create --name extract multiarch/qemu-user-static`);
+        shellExec(`podman ps -a`);
+        shellExec(`sudo podman cp extract:/usr/bin/qemu-aarch64-static ${nfsHostPath}/usr/bin/`);
+        shellExec(`sudo podman rm extract`);
+        shellExec(`podman ps -a`);
+
+        switch (host) {
+          case 'rpi4mb':
+            shellExec(`file ${nfsHostPath}/bin/bash`); // expected: ELF 64-bit LSB pie executable, ARM aarch64 …
+            break;
+
+          default:
+            break;
+        }
+
+        shellExec(`sudo chroot ${nfsHostPath} /usr/bin/qemu-aarch64-static /bin/bash <<'EOF'
+/debootstrap/debootstrap --second-stage
+EOF`);
+      }
+      if (process.argv.includes('mount')) {
+        shellExec(`sudo mount --bind /proc ${nfsHostPath}/proc`);
+        shellExec(`sudo mount --bind /sys  ${nfsHostPath}/sys`);
+        shellExec(`sudo mount --rbind /dev  ${nfsHostPath}/dev`);
+      }
+
+      if (process.argv.includes('build')) {
+        switch (host) {
+          case 'rpi4mb':
+            const ipaddr = process.env.RPI4_IP;
+
+            await updateVirtualRoot({
+              IP_ADDRESS,
+              architecture,
+              host,
+              nfsHostPath,
+              ipaddr,
+            });
+
+            break;
+
+          default:
+            break;
+        }
+      }
+      // if (process.argv.includes('mount')) {
+      //   shellExec(`sudo mount --bind /lib/modules ${nfsHostPath}/lib/modules`);
+      // }
+
+      break;
+    }
+
+    case 'close-virtual-root': {
+      const architecture = process.argv[3];
+      const host = process.argv[4];
+      const nfsHostPath = `${process.env.NFS_EXPORT_PATH}/${host}`;
+      shellExec(`sudo umount ${nfsHostPath}/proc`);
+      shellExec(`sudo umount ${nfsHostPath}/sys`);
+      shellExec(`sudo umount ${nfsHostPath}/dev`);
+      // shellExec(`sudo umount ${nfsHostPath}/lib/modules`);
+      break;
+    }
+
+    case 'mount': {
+      const mounts = shellExec(`mount`).split(`\n`);
+      console.table(
+        mounts
+          .filter((l) => l.trim())
+          .map(
+            (o) => (
+              (o = o.split(' ')),
+              {
+                path: o[2],
+                type: o[4],
+                permissions: o[5],
+              }
+            ),
+          ),
+      );
+      break;
+    }
+
+    case 'create-ports': {
+      const cmd = [];
+      const ipaddr = getLocalIPv4Address();
+      for (const port of ['5240']) {
+        const name = 'maas';
+        cmd.push(`${name}:${port}-${port}:${ipaddr}`);
+      }
+      pbcopy(`node engine-private/r create-port ${cmd}`);
+      break;
+    }
+
+    case 'maas-ports': {
+      // Configure firewall:
+
+      // systemctl stop firewalld
+      // systemctl mask firewalld
+
+      // ufw disable
+      // ufw enable
+
+      // sudo snap install ufw
+      // const ports = ['80', '443', '22', '3000-3100'];
+      const ports = [
+        '43',
+        '53',
+        '60',
+        '66',
+        '67',
+        '69',
+        '4011',
+        '111',
+        '2049',
+        '20048',
+        '20049',
+        '32765',
+        '32766',
+        '5248',
+        '5240',
+      ];
+      for (const port of ports) {
+        shellExec(`ufw allow ${port}/tcp`);
+        shellExec(`ufw allow ${port}/udp`);
+      }
+
+      shellExec(`sudo systemctl mask firewalld`);
+
+      break;
+    }
+
+    case 'iptables': {
+      shellExec(`sudo systemctl enable nftables`);
+      shellExec(`sudo systemctl restart nftables`);
+
+      shellExec(`sudo tee /etc/nftables.conf <<EOF
+table inet filter {
+  chain input {
+    type filter hook input priority 0;
+    policy drop;
+    tcp dport 22 accept
+  }
+}
+EOF`);
+      shellExec(`sudo nft -f /etc/nftables.conf`);
+
+      // sudo systemctl stop nftables
+      // sudo systemctl disable nftables
+
+      break;
+    }
+
+    case 'rpi4': {
+      // Rpi4 Run Bootloader:
+
+      // 1) create boot.conf
+
+      // 2) Run lite RPiOs from rpi-imager
+      // with boot.conf files in root disk path
+
+      // 3) cd /boot/firmware && sudo rpi-eeprom-config --apply boot.conf
+
+      // 4) sudo reboot
+
+      // 5) check: 'vcgencmd bootloader_version'
+      // 6) check: 'vcgencmd bootloader_config'
+
+      // 7) shutdown and restart without sd card
+
+      // sudo apt update
+      // sudo apt install git
+
+      break;
+    }
+
+    case 'blue': {
+      // lsusb | grep blue -i
+      // rfkill list
+      // sudo service bluetooth start
+      // bluetoothctl show
+      // sudo rfkill unblock bluetooth
+      // dmesg | grep -i bluetooth
+      // journalctl -u bluetooth -f
+      // sudo dnf update bluez bluez-libs bluez-utils
+      // sudo rmmod btusb
+      // sudo modprobe btusb
+      break;
+    }
+
+    case 'fastapi-models': {
+      shellExec(`chmod +x ../full-stack-fastapi-template/backend/initial_data.sh`);
+      shellExec(`../full-stack-fastapi-template/backend/initial_data.sh`);
+      shellExec(`../full-stack-fastapi-template/backend/initial_data.sh`);
+      break;
+    }
+
+    case 'fastapi': {
+      // node bin/deploy fastapi reset
+      // node bin/deploy fastapi reset build-back build-front secret run-back run-front
+      // https://github.com/NonsoEchendu/full-stack-fastapi-project
+      // https://github.com/fastapi/full-stack-fastapi-template
+      const path = `../full-stack-fastapi-template`;
+      const VITE_API_URL = `http://localhost:8000`;
+
+      if (process.argv.includes('reset')) shellExec(`sudo rm -rf ${path}`);
+
+      if (!fs.existsSync(path))
+        shellExec(`cd .. && git clone https://github.com/fastapi/full-stack-fastapi-template.git`);
+
+      shellExec(`cd ${path} && git checkout . && git clean -f -d`);
+      const password = fs.readFileSync(`/home/dd/engine/engine-private/postgresql-password`, 'utf8');
+
+      fs.writeFileSync(
+        `${path}/.env`,
+        fs
+          .readFileSync(`${path}/.env`, 'utf8')
+          .replace(`FIRST_SUPERUSER=admin@example.com`, `FIRST_SUPERUSER=development@underpost.net`)
+          .replace(`FIRST_SUPERUSER_PASSWORD=changethis`, `FIRST_SUPERUSER_PASSWORD=${password}`)
+          .replace(`SECRET_KEY=changethis`, `SECRET_KEY=${password}`)
+          .replace(`POSTGRES_DB=app`, `POSTGRES_DB=postgresdb`)
+          .replace(`POSTGRES_USER=postgres`, `POSTGRES_USER=admin`)
+          .replace(`POSTGRES_PASSWORD=changethis`, `POSTGRES_PASSWORD=${password}`),
+        'utf8',
+      );
+      fs.writeFileSync(
+        `${path}/backend/app/core/db.py`,
+        fs
+          .readFileSync(`${path}/backend/app/core/db.py`, 'utf8')
+          .replace(`    # from sqlmodel import SQLModel`, `    from sqlmodel import SQLModel`)
+          .replace(`   # SQLModel.metadata.create_all(engine)`, `   SQLModel.metadata.create_all(engine)`),
+
+        'utf8',
+      );
+
+      fs.copySync(`./manifests/deployment/fastapi/initial_data.sh`, `${path}/backend/initial_data.sh`);
+
+      fs.writeFileSync(
+        `${path}/frontend/Dockerfile`,
+        fs
+          .readFileSync(`${path}/frontend/Dockerfile`, 'utf8')
+          .replace('ARG VITE_API_URL=${VITE_API_URL}', `ARG VITE_API_URL='${VITE_API_URL}'`),
+        'utf8',
+      );
+
+      fs.writeFileSync(
+        `${path}/frontend/.env`,
+        fs
+          .readFileSync(`${path}/frontend/.env`, 'utf8')
+          .replace(`VITE_API_URL=http://localhost:8000`, `VITE_API_URL=${VITE_API_URL}`)
+          .replace(`MAILCATCHER_HOST=http://localhost:1080`, `MAILCATCHER_HOST=http://localhost:1081`),
+
+        'utf8',
+      );
+
+      if (process.argv.includes('models')) {
+        shellExec(`node bin/deploy fastapi-models`);
+        break;
+      }
+
+      if (process.argv.includes('build-back')) {
+        const imageName = `fastapi-backend:latest`;
+        shellExec(`sudo podman pull docker.io/library/python:3.10`);
+        shellExec(`sudo podman pull ghcr.io/astral-sh/uv:0.5.11`);
+        shellExec(`sudo rm -rf ${path}/${imageName.replace(':', '_')}.tar`);
+        const args = [
+          `node bin dockerfile-image-build --path ${path}/backend/`,
+          `--image-name=${imageName} --image-path=${path}`,
+          `--podman-save --${process.argv.includes('kubeadm') ? 'kubeadm' : 'kind'}-load --no-cache`,
+        ];
+        shellExec(args.join(' '));
+      }
+      if (process.argv.includes('build-front')) {
+        const imageName = `fastapi-frontend:latest`;
+        shellExec(`sudo podman pull docker.io/library/node:20`);
+        shellExec(`sudo podman pull docker.io/library/nginx:1`);
+        shellExec(`sudo rm -rf ${path}/${imageName.replace(':', '_')}.tar`);
+        const args = [
+          `node bin dockerfile-image-build --path ${path}/frontend/`,
+          `--image-name=${imageName} --image-path=${path}`,
+          `--podman-save --${process.argv.includes('kubeadm') ? 'kubeadm' : 'kind'}-load --no-cache`,
+        ];
+        shellExec(args.join(' '));
+      }
+      if (process.argv.includes('secret')) {
+        {
+          const secretSelector = `fastapi-postgres-credentials`;
+          shellExec(`sudo kubectl delete secret ${secretSelector}`);
+          shellExec(
+            `sudo kubectl create secret generic ${secretSelector}` +
+              ` --from-literal=POSTGRES_DB=postgresdb` +
+              ` --from-literal=POSTGRES_USER=admin` +
+              ` --from-file=POSTGRES_PASSWORD=/home/dd/engine/engine-private/postgresql-password`,
+          );
+        }
+        {
+          const secretSelector = `fastapi-backend-config-secret`;
+          shellExec(`sudo kubectl delete secret ${secretSelector}`);
+          shellExec(
+            `sudo kubectl create secret generic ${secretSelector}` +
+              ` --from-file=SECRET_KEY=/home/dd/engine/engine-private/postgresql-password` +
+              ` --from-literal=FIRST_SUPERUSER=development@underpost.net` +
+              ` --from-file=FIRST_SUPERUSER_PASSWORD=/home/dd/engine/engine-private/postgresql-password`,
+          );
+        }
+      }
+      if (process.argv.includes('run-back')) {
+        shellExec(`sudo kubectl apply -f ./manifests/deployment/fastapi/backend-deployment.yml`);
+        shellExec(`sudo kubectl apply -f ./manifests/deployment/fastapi/backend-service.yml`);
+      }
+      if (process.argv.includes('run-front')) {
+        shellExec(`sudo kubectl apply -f ./manifests/deployment/fastapi/frontend-deployment.yml`);
+        shellExec(`sudo kubectl apply -f ./manifests/deployment/fastapi/frontend-service.yml`);
+      }
+      break;
+    }
+
+    case 'conda': {
+      // set -e
+      // ENV_NAME="${1:-cuda_env}"
+      // eval "$(conda shell.bash hook)"
+      // conda activate "${ENV_NAME}"
+      shellExec(
+        `export PATH="/root/miniconda3/bin:$PATH" && conda init && conda config --set auto_activate_base false`,
+      );
+      shellExec(`conda env list`);
+      break;
+    }
+
+    case 'kafka': {
+      // https://medium.com/@martin.hodges/deploying-kafka-on-a-kind-kubernetes-cluster-for-development-and-testing-purposes-ed7adefe03cb
+      const imageName = `doughgle/kafka-kraft`;
+      shellExec(`docker pull ${imageName}`);
+      if (!process.argv.includes('kubeadm'))
+        shellExec(
+          `${process.argv.includes('kubeadm') ? `ctr -n k8s.io images import` : `kind load docker-image`} ${imageName}`,
+        );
+      shellExec(`kubectl create namespace kafka`);
+      shellExec(`kubectl apply -f ./manifests/deployment/kafka/deployment.yaml`);
+      // kubectl logs kafka-0 -n kafka | grep STARTED
+      // kubectl logs kafka-1 -n kafka | grep STARTED
+      // kubectl logs kafka-2 -n kafka | grep STARTED
+
+      // kafka-topics.sh --create --topic my-topic --bootstrap-server kafka-svc:9092
+      // kafka-topics.sh --list --topic my-topic --bootstrap-server kafka-svc:9092
+      // kafka-topics.sh --delete --topic my-topic --bootstrap-server kafka-svc:9092
+
+      // kafka-console-producer.sh --bootstrap-server kafka-svc:9092 --topic my-topic
+      // kafka-console-consumer.sh --bootstrap-server kafka-svc:9092 --topic my-topic
+      break;
+    }
+
+    case 'nvidia-gpu-operator': {
+      // https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
+      shellExec(`curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | \
+sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo`);
+
+      const NVIDIA_CONTAINER_TOOLKIT_VERSION = '1.17.8-1';
+
+      shellExec(`sudo dnf install -y \
+nvidia-container-toolkit-${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
+nvidia-container-toolkit-base-${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
+libnvidia-container-tools-${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
+libnvidia-container1-${NVIDIA_CONTAINER_TOOLKIT_VERSION}`);
+
+      // https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html
+
+      shellExec(`kubectl create ns gpu-operator`);
+      shellExec(`kubectl label --overwrite ns gpu-operator pod-security.kubernetes.io/enforce=privileged`);
+
+      shellExec(`helm repo add nvidia https://helm.ngc.nvidia.com/nvidia \
+    && helm repo update`);
+
+      //       shellExec(`helm install --wait --generate-name \
+      // -n gpu-operator --create-namespace \
+      // nvidia/gpu-operator \
+      // --version=v25.3.1 \
+      // --set toolkit.version=v1.16.1-ubi8`);
+
+      shellExec(`helm install --wait --generate-name \
+-n gpu-operator --create-namespace \
+nvidia/gpu-operator \
+--version=v25.3.1 \
+--set driver.enabled=false \
+--set driver.repository=nvcr.io/nvidia \
+--set cdi.enabled=true \
+--set cdi.default=true \
+--set toolkit.env[0].name=CONTAINERD_CONFIG \
+--set toolkit.env[0].value=/etc/containerd/config.toml \
+--set toolkit.env[1].name=CONTAINERD_SOCKET \
+--set toolkit.env[1].value=/run/containerd/containerd.sock \
+--set toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS \
+--set toolkit.env[2].value=nvidia \
+--set-string toolkit.env[3].name=CONTAINERD_SET_AS_DEFAULT \
+--set-string toolkit.env[3].value=true`);
+
+      // Check gpu drivers
+      shellExec(
+        `break;kubectl get nodes -o json | jq '.items[].metadata.labels | keys | any(startswith("feature.node.kubernetes.io"))'`,
+      );
+      break;
+    }
+
+    case 'kubeflow-spark-operator': {
+      // Use case:
+      // Data Processing Pipelines: Used for ETL tasks where Spark can handle large data volumes efficiently.
+      // Real-Time Analytics: Processing data from streaming sources (e.g., Kafka) for real-time analytics.
+      // Machine Learning and Data Science: Training and deploying machine learning models at scale using Spark MLlib.
+
+      shellExec(`helm repo add spark-operator https://kubeflow.github.io/spark-operator`);
+      shellExec(`helm install spark-operator spark-operator/spark-operator \
+  --namespace spark-operator \
+  --create-namespace \
+  --wait`);
+
+      const image = `spark:3.5.5`;
+      shellExec(`sudo docker pull ${image}`);
+      if (!process.argv.includes('kubeadm'))
+        shellExec(
+          `sudo ${
+            process.argv.includes('kubeadm') ? `ctr -n k8s.io images import` : `kind load docker-image`
+          } ${image}`,
+        );
+      shellExec(`kubectl apply -f ./manifests/deployment/spark/spark-pi-py.yaml`);
+
+      // Check the status of the Spark job:
+      // kubectl get sparkapplications.sparkoperator.k8s.io -n default
+      // kubectl get sparkapplication
+
+      // Check case log:
+      // kubectl logs -f spark-pi-python-driver
+      // kubectl logs -f spark-pi-python-driver | grep Pi
+      // kubectl describe sparkapplication spark-gpu-test
+
+      // Uninstall:
+      // kubectl delete sparkapplications.sparkoperator.k8s.io spark-pi-python -n default
+      // helm delete spark-operator -n spark-operator
+
+      // Gpu plugins:
+      // https://github.com/NVIDIA/spark-rapids
+      // RAPIDS Accelerator
+      break;
+    }
+
+    case 'sbt': {
+      // https://www.scala-sbt.org/1.x/docs/Installing-sbt-on-Linux.html
+
+      // sudo rm -f /etc/yum.repos.d/bintray-rpm.repo
+      // curl -L https://www.scala-sbt.org/sbt-rpm.repo > sbt-rpm.repo
+      // sudo mv sbt-rpm.repo /etc/yum.repos.d/
+      // sudo yum install sbt
+      break;
+    }
   }
 } catch (error) {
   logger.error(error, error.stack);