underpost 2.8.82 → 2.8.84

package/src/cli/run.js

@@ -0,0 +1,262 @@
+import { daemonProcess, getTerminalPid, openTerminal, pbcopy, shellCd, shellExec } from '../server/process.js';
+import read from 'read';
+import { getNpmRootPath } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import UnderpostTest from './test.js';
+import fs from 'fs-extra';
+import { range, setPad, timer } from '../client/components/core/CommonJs.js';
+import UnderpostDeploy from './deploy.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostRun {
+  static DEFAULT_OPTION = {
+    dev: false,
+    podName: '',
+    volumeHostPath: '',
+    volumeMountPath: '',
+    imageName: '',
+    containerName: '',
+    namespace: '',
+  };
+  static RUNNERS = {
+    'spark-template': (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      const dir = '/home/dd/spark-template';
+      shellExec(`sudo rm -rf ${dir}`);
+      shellCd('/home/dd');
+
+      // pbcopy(`cd /home/dd && sbt new underpostnet/spark-template.g8`);
+      // await read({ prompt: 'Command copy to clipboard, press enter to continue.\n' });
+      shellExec(`cd /home/dd && sbt new underpostnet/spark-template.g8 '--name=spark-template'`);
+
+      shellCd(dir);
+
+      shellExec(`git init && git add . && git commit -m "Base implementation"`);
+      shellExec(`chmod +x ./replace_params.sh`);
+      shellExec(`chmod +x ./build.sh`);
+
+      shellExec(`./replace_params.sh`);
+      shellExec(`./build.sh`);
+
+      shellCd('/home/dd/engine');
+    },
+    rmi: (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      shellExec(`podman rmi $(podman images -qa) --force`);
+    },
+    kill: (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      shellExec(`sudo kill -9 $(lsof -t -i:${path})`);
+    },
+    secret: (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      shellExec(
+        `underpost secret underpost --create-from-file ${
+          path ? path : `/home/dd/engine/engine-private/conf/dd-cron/.env.production`
+        }`,
+      );
+    },
+    'gpu-env': (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      shellExec(
+        `node bin cluster --dev --reset && node bin cluster --dev --dedicated-gpu --kubeadm && kubectl get pods --all-namespaces -o wide -w`,
+      );
+    },
+    'tf-gpu-test': (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      const { underpostRoot } = options;
+      shellExec(`kubectl delete configmap tf-gpu-test-script`);
+      shellExec(`kubectl delete pod tf-gpu-test-pod`);
+      shellExec(`kubectl apply -f ${underpostRoot}/manifests/deployment/tensorflow/tf-gpu-test.yaml`);
+    },
+    ide: (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      const { underpostRoot } = options;
+      shellExec(`node ${underpostRoot}/bin/vs ${path}`);
+    },
+    monitor: (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      const pid = getTerminalPid();
+      logger.info('monitor pid', pid);
+      const checkPath = '/await';
+      const _monitor = async () => {
+        const result = UnderpostDeploy.API.existsContainerFile({ podName: path, path: checkPath });
+        logger.info('monitor', result);
+        if (result === true) {
+          switch (path) {
+            case 'tf-vae-test':
+              {
+                const nameSpace = 'default';
+                const podName = path;
+                const basePath = '/home/dd';
+                const scriptPath = '/site/en/tutorials/generative/cvae.py';
+                // shellExec(
+                //   `sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${scriptPath} ${basePath}/lab/src/${scriptPath
+                //     .split('/')
+                //     .pop()}`,
+                // );
+                // const file = fs.readFileSync(`${basePath}/lab/src/${scriptPath.split('/').pop()}`, 'utf8');
+                //                 fs.writeFileSync(
+                //                   `${basePath}/lab/src/${scriptPath.split('/').pop()}`,
+                //                   file.replace(
+                //                     `import time`,
+                //                     `import time
+                // print('=== SCRIPT UPDATE TEST ===')`,
+                //                   ),
+                //                   'utf8',
+                //                 );
+                shellExec(
+                  `sudo kubectl cp ${basePath}/lab/src/${scriptPath
+                    .split('/')
+                    .pop()} ${nameSpace}/${podName}:${basePath}/docs${scriptPath}`,
+                );
+                // shellExec(`sudo kubectl exec -i ${podName} -- sh -c "ipython ${basePath}/docs${scriptPath}"`);
+                shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf ${checkPath}"`);
+
+                {
+                  const checkPath = `/latent_space_plot.png`;
+                  const outsPaths = [];
+                  logger.info('monitor', checkPath);
+                  while (!UnderpostDeploy.API.existsContainerFile({ podName, path: `/home/dd/docs${checkPath}` }))
+                    await timer(1000);
+
+                  {
+                    const toPath = `${basePath}/lab${checkPath}`;
+                    outsPaths.push(toPath);
+                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${checkPath} ${toPath}`);
+                  }
+
+                  for (let i of range(1, 10)) {
+                    i = `/image_at_epoch_${setPad(i, '0', 4)}.png`;
+                    const toPath = `${basePath}/lab/${i}`;
+                    outsPaths.push(toPath);
+                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${i} ${toPath}`);
+                  }
+                  openTerminal(`firefox ${outsPaths.join(' ')}`, { single: true });
+                }
+                shellExec(`sudo kill -9 ${pid}`);
+              }
+              break;
+
+            default:
+              break;
+          }
+          return;
+        }
+        await timer(1000);
+        _monitor();
+      };
+      _monitor();
+    },
+    'tf-vae-test': async (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      const { underpostRoot } = options;
+      const podName = 'tf-vae-test';
+      await UnderpostRun.RUNNERS['deploy-job']('', {
+        podName,
+        // volumeMountPath: '/custom_images',
+        // volumeHostPath: '/home/dd/engine/src/client/public/cyberia/assets/skin',
+        on: {
+          init: async () => {
+            openTerminal(`node bin run --dev monitor ${podName}`);
+          },
+        },
+        args: [
+          `pip install --upgrade \
+               nbconvert \
+               tensorflow-probability==0.23.0 \
+               imageio \
+               git+https://github.com/tensorflow/docs \
+               matplotlib \
+               "numpy<1.25,>=1.21"`,
+          'mkdir -p /home/dd',
+          'cd /home/dd',
+          'git clone https://github.com/tensorflow/docs.git',
+          'cd docs',
+          'jupyter nbconvert --to python site/en/tutorials/generative/cvae.ipynb',
+          `echo '' > /await`,
+          `echo '=== WAITING SCRIPT LAUNCH ==='`,
+          `while [ -f /await ]; do sleep 1; done`,
+          `echo '=== FINISHED ==='`,
+          daemonProcess(`ipython site/en/tutorials/generative/cvae.py`),
+        ],
+      });
+    },
+    'deploy-job': async (path, options = UnderpostRun.DEFAULT_OPTION) => {
+      const podName = options.podName || 'deploy-job';
+      const volumeName = `${podName}-volume`;
+      const args = (options.args ? options.args : path ? [`python ${path}`] : []).filter((c) => c.trim());
+      const imageName = options.imageName || 'nvcr.io/nvidia/tensorflow:24.04-tf2-py3';
+      const containerName = options.containerName || `${podName}-container`;
+      const gpuEnable = imageName.match('nvidia');
+      const runtimeClassName = gpuEnable ? 'nvidia' : '';
+      const namespace = options.namespace || 'default';
+      const volumeMountPath = options.volumeMountPath || path;
+      const volumeHostPath = options.volumeHostPath || path;
+      const enableVolumeMount = volumeHostPath && volumeMountPath;
+
+      const cmd = `kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${podName}
+  namespace: ${namespace}
+spec:
+  restartPolicy: Never
+${runtimeClassName ? `  runtimeClassName: ${runtimeClassName}` : ''}
+  containers:
+    - name: ${containerName}
+      image: ${imageName}
+      imagePullPolicy: IfNotPresent
+      tty: true
+      stdin: true
+      command: ${JSON.stringify(options.command ? options.command : ['/bin/bash', '-c'])}
+${
+  args.length > 0
+    ? `      args:
+        - |
+${args.map((arg) => `          ${arg}`).join('\n')}`
+    : ''
+}
+${
+  gpuEnable
+    ? `      resources:
+        limits:
+          nvidia.com/gpu: '1'
+      env:
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: all`
+    : ''
+}
+${
+  enableVolumeMount
+    ? `
+      volumeMounts:
+        - name: ${volumeName}
+          mountPath: ${volumeMountPath}
+  volumes:
+    - name: ${volumeName}
+      hostPath:
+        path: ${volumeHostPath}
+        type: ${fs.statSync(volumeHostPath).isDirectory() ? 'Directory' : 'File'}`
+    : ''
+}
+EOF`;
+      shellExec(`kubectl delete pod ${podName}`);
+      console.log(cmd);
+      shellExec(cmd, { disableLog: true });
+      const successInstance = await UnderpostTest.API.statusMonitor(podName);
+      if (successInstance) {
+        options.on?.init ? await options.on.init() : null;
+        shellExec(`kubectl logs -f ${podName}`);
+      }
+    },
+  };
+  static API = {
+    async callback(runner, path, options = UnderpostRun.DEFAULT_OPTION) {
+      const npmRoot = getNpmRootPath();
+      const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      if (options.command) options.command = options.command.split(',');
+      if (options.args) options.args = options.args.split(',');
+      options.underpostRoot = underpostRoot;
+      options.npmRoot = npmRoot;
+      logger.info('callback', { path, options });
+      const result = await UnderpostRun.RUNNERS[runner](path, options);
+      return result;
+    },
+  };
+}
+
+export default UnderpostRun;

package/src/cli/test.js

@@ -94,7 +94,7 @@ class UnderpostTest {
         const _monitor = async () => {
           await timer(deltaMs);
           const pods = UnderpostDeploy.API.get(podName, kindType);
-          const result = pods.find((p) => p.STATUS === status);
+          let result = pods.find((p) => p.STATUS === status || (status === 'Running' && p.STATUS === 'Completed'));
           logger.info(
             `Testing pod ${podName}... ${result ? 1 : 0}/1 - elapsed time ${deltaMs * (index + 1)}s - attempt ${
               index + 1

package/src/index.js

@@ -16,6 +16,7 @@ import UnderpostImage from './cli/image.js';
 import UnderpostLxd from './cli/lxd.js';
 import UnderpostMonitor from './cli/monitor.js';
 import UnderpostRepository from './cli/repository.js';
+import UnderpostRun from './cli/run.js';
 import UnderpostScript from './cli/script.js';
 import UnderpostSecret from './cli/secrets.js';
 import UnderpostTest from './cli/test.js';
@@ -33,102 +34,102 @@ class Underpost {
    * @type {String}
    * @memberof Underpost
    */
-  static version = 'v2.8.82';
+  static version = 'v2.8.84';
   /**
    * Repository cli API
    * @static
-   * @type {UnderpostRepository}
+   * @type {UnderpostRepository.API}
    * @memberof Underpost
    */
   static repo = UnderpostRepository.API;
   /**
    * Root Env cli API
    * @static
-   * @type {UnderpostRootEnv}
+   * @type {UnderpostRootEnv.API}
    * @memberof Underpost
    */
   static env = UnderpostRootEnv.API;
   /**
    * Test cli API
    * @static
-   * @type {UnderpostTest}
+   * @type {UnderpostTest.API}
    * @memberof Underpost
    */
   static test = UnderpostTest.API;
   /**
    * Underpost Start Up cli API
    * @static
-   * @type {UnderpostStartUp}
+   * @type {UnderpostStartUp.API}
    * @memberof Underpost
    */
   static start = UnderpostStartUp.API;
   /**
    * Cluster cli API
    * @static
-   * @type {UnderpostCluster}
+   * @type {UnderpostCluster.API}
    * @memberof Underpost
    */
   static cluster = UnderpostCluster.API;
   /**
    * Image cli API
    * @static
-   * @type {UnderpostImage}
+   * @type {UnderpostImage.API}
    * @memberof Underpost
    */
   static image = UnderpostImage.API;
   /**
    * Secrets cli API
    * @static
-   * @type {UnderpostSecret}
+   * @type {UnderpostSecret.API}
    * @memberof Underpost
    */
   static secret = UnderpostSecret.API;
   /**
    * Scripts cli API
    * @static
-   * @type {UnderpostScript}
+   * @type {UnderpostScript.API}
    * @memberof Underpost
    */
   static script = UnderpostScript.API;
   /**
    * Database cli API
    * @static
-   * @type {UnderpostDB}
+   * @type {UnderpostDB.API}
    * @memberof Underpost
    */
   static db = UnderpostDB.API;
   /**
    * Deployment cli API
    * @static
-   * @type {UnderpostDeploy}
+   * @type {UnderpostDeploy.API}
    * @memberof Underpost
    */
   static deploy = UnderpostDeploy.API;
   /**
    * Cron cli API
    * @static
-   * @type {UnderpostCron}
+   * @type {UnderpostCron.API}
    * @memberof Underpost
    */
   static cron = UnderpostCron.API;
   /**
    * File Storage cli API
    * @static
-   * @type {UnderpostFileStorage}
+   * @type {UnderpostFileStorage.API}
    * @memberof Underpost
    */
   static fs = UnderpostFileStorage.API;
   /**
    * Monitor cli API
    * @static
-   * @type {UnderpostMonitor}
+   * @type {UnderpostMonitor.API}
    * @memberof Underpost
    */
   static monitor = UnderpostMonitor.API;
   /**
    * LXD cli API
    * @static
-   * @type {UnderpostLxd}
+   * @type {UnderpostLxd.API}
    * @memberof Underpost
    */
   static lxd = UnderpostLxd.API;
@@ -136,15 +137,23 @@ class Underpost {
   /**
    * Cloud Init cli API
    * @static
-   * @type {UnderpostCloudInit}
+   * @type {UnderpostCloudInit.API}
    * @memberof Underpost
    */
   static cloudInit = UnderpostCloudInit.API;
 
+  /**
+   * Run cli API
+   * @static
+   * @type {UnderpostRun.API}
+   * @memberof Underpost
+   */
+  static run = UnderpostRun.API;
+
   /**
    * Baremetal cli API
    * @static
-   * @type {UnderpostBaremetal}
+   * @type {UnderpostBaremetal.API}
    * @memberof Underpost
    */
   static baremetal = UnderpostBaremetal.API;

package/src/server/conf.js

@@ -1171,7 +1171,7 @@ const writeEnv = (envPath, envObj) =>
     'utf8',
   );
 
-const buildCliDoc = (program) => {
+const buildCliDoc = (program, oldVersion, newVersion) => {
   let md = shellExec(`node bin help`, { silent: true, stdout: true }).split('Options:');
   const baseOptions =
     `## ${md[0].split(`\n`)[2]}
@@ -1207,13 +1207,15 @@ const buildCliDoc = (program) => {
       `
   `;
   });
+  md = md.replaceAll(oldVersion, newVersion);
   fs.writeFileSync(`./src/client/public/nexodev/docs/references/Command Line Interface.md`, md, 'utf8');
   fs.writeFileSync(`./cli.md`, md, 'utf8');
   const readmeSplit = `pwa-microservices-template</a>`;
   const readme = fs.readFileSync(`./README.md`, 'utf8').split(readmeSplit);
   fs.writeFileSync(
     './README.md',
-    readme[0] +
+    (
+      readme[0] +
       readmeSplit +
       `
 
@@ -1223,7 +1225,8 @@ const buildCliDoc = (program) => {
       
 <a target="_top" href="https://github.com/underpostnet/pwa-microservices-template/blob/master/cli.md">See complete CLI Docs here.</a>
       
-`,
+`
+    ).replaceAll(oldVersion, newVersion),
     'utf8',
   );
 };

package/src/server/process.js

@@ -10,8 +10,6 @@ dotenv.config();
 
 const logger = loggerFactory(import.meta);
 
-// process.exit();
-
 const getRootDirectory = () => process.cwd().replace(/\\/g, '/');
 
 const ProcessController = {
@@ -63,27 +61,26 @@ const shellCd = (cd, options = { disableLog: false }) => {
   return shell.cd(cd);
 };
 
-function pbcopy(data) {
-  switch (process.platform) {
-    case 'linux':
-      {
-        // sudo dnf install xclip
-        // sudo apt update
-        // sudo apt install xclip
-        // paste: xclip -o
-        // copy:
-        // shellExec(`echo "${data}" | xclip -sel clip`, { async: true });
-      }
+const openTerminal = (cmd, options = { single: false }) => {
+  if (options.single === true) {
+    shellExec(`setsid gnome-terminal -- bash -ic "${cmd}; exec bash" >/dev/null 2>&1 &`);
+    return;
+  }
+  shellExec(`gnome-terminal -- bash -c "${cmd}; exec bash" & disown`, {
+    async: true,
+    stdout: true,
+  });
+};
 
-      break;
+const daemonProcess = (cmd) => `exec bash -c '${cmd}; exec tail -f /dev/null'`;
 
-    default:
-      break;
-  }
+// list all terminals: pgrep gnome-terminal
+// list last terminal: pgrep -n gnome-terminal
+const getTerminalPid = () => JSON.parse(shellExec(`pgrep -n gnome-terminal`, { stdout: true, silent: true }));
 
+function pbcopy(data) {
   clipboard.writeSync(data || '🦄');
-
   logger.info(`copied to clipboard`, clipboard.readSync());
 }
 
-export { ProcessController, getRootDirectory, shellExec, shellCd, pbcopy };
+export { ProcessController, getRootDirectory, shellExec, shellCd, pbcopy, openTerminal, getTerminalPid, daemonProcess };