underpost 2.8.852 → 2.8.854

package/cli.md

@@ -1,4 +1,4 @@
-## underpost ci/cd cli v2.8.852
+## underpost ci/cd cli v2.8.854
 
 ### Usage: `underpost [options] [command]`
   ```
@@ -23,6 +23,7 @@ Commands:
   dockerfile-pull-base-images [options]                      Pulls required Underpost Dockerfile base images and optionally loads them into clusters.
   install                                                    Quickly imports Underpost npm dependencies by copying them.
   db [options] <deploy-list>                                 Manages database operations, including import, export, and collection management.
+  metadata [options] [deploy-id] [host] [path]               Manages cluster metadata operations, including import and export.
   script [options] <operator> <script-name> [script-value]   Supports a variety of built-in Underpost global scripts, their preset lifecycle events, and arbitrary custom scripts.
   cron [options] [deploy-list] [job-list]                    Manages cron jobs, including initialization, execution, and configuration updates.
   fs [options] [path]                                        Manages file storage, defaulting to file upload operations.
@@ -235,6 +236,9 @@ Options:
                           specified namespace.
   --kubeadm               Initializes the cluster using kubeadm for control
                           plane management.
+  --grafana               Initializes the cluster with a Grafana deployment.
+  --prom [hosts]          Initializes the cluster with a Prometheus Operator
+                          deployment and monitor scrap for specified hosts.
   --dev                   Initializes a development-specific cluster
                           configuration.
   --list-pods             Displays detailed information about all pods.
@@ -288,8 +292,6 @@ Options:
   --build-manifest                  Builds Kubernetes YAML manifests, including
                                     deployments, services, proxies, and
                                     secrets.
-  --dashboard-update                Updates dashboard instance data with the
-                                    current router configuration.
   --replicas <replicas>             Sets a custom number of replicas for
                                     deployments.
   --versions <deployment-versions>  A comma-separated list of custom deployment
@@ -302,9 +304,6 @@ Options:
   --kubeadm                         Enables the kubeadm context for deployment
                                     operations.
   --restore-hosts                   Restores default `/etc/hosts` entries.
-  --rebuild-clients-bundle          Inside the container, rebuilds client
-                                    bundles (only static public or storage
-                                    client files).
   -h, --help                        display help for command
  
 ```
@@ -427,6 +426,27 @@ Options:
 ```
   
 
+### `metadata` :
+```
+ Usage: underpost metadata [options] [deploy-id] [host] [path]
+
+Manages cluster metadata operations, including import and export.
+
+Arguments:
+  deploy-id    The deployment ID to manage metadata.
+  host         The host to manage metadata.
+  path         The path to manage metadata.
+
+Options:
+  --import     Imports from local storage.
+  --export     Exports to local storage.
+  --crons      Apply to cron data collection
+  --instances  Apply to instance data collection
+  -h, --help   display help for command
+ 
+```
+  
+
 ### `script` :
 ```
  Usage: underpost script [options] <operator> <script-name> [script-value]
@@ -461,19 +481,16 @@ Manages cron jobs, including initialization, execution, and configuration
 updates.
 
 Arguments:
-  deploy-list         A comma-separated list of deployment IDs (e.g.,
-                      "default-a,default-b").
-  job-list            A comma-separated list of job IDs. Options: callback,
-                      updateDashboardData. Defaults to all available jobs.
+  deploy-list  A comma-separated list of deployment IDs (e.g.,
+               "default-a,default-b").
+  job-list     A comma-separated list of job IDs. Options: callback. Defaults
+               to all available jobs.
 
 Options:
-  --itc               Executes cron jobs within the container execution
-                      context.
-  --init              Initializes cron jobs for the default deployment ID.
-  --git               Uploads cron job configurations to GitHub.
-  --dashboard-update  Updates dashboard cron data with the current job
-                      configurations.
-  -h, --help          display help for command
+  --itc        Executes cron jobs within the container execution context.
+  --init       Initializes cron jobs for the default deployment ID.
+  --git        Uploads cron job configurations to GitHub.
+  -h, --help   display help for command
  
 ```
   
@@ -574,7 +591,7 @@ Options:
 Runs a script from the specified path.
 
 Arguments:
-  runner-id                                The runner ID to run. Options: spark-template, rmi, kill, secret, gpu-env, tf-gpu-test, dev-cluster, cyberia-ide, engine-ide, template-deploy, ssh-deploy, ide, monitor, db-client, cluster, deploy, tf-vae-test, deploy-job.
+  runner-id                                The runner ID to run. Options: spark-template, rmi, kill, secret, gpu-env, tf-gpu-test, dev-cluster, ssh-cluster-info, cyberia-ide, engine-ide, template-deploy, clean, upgrade, ssh-deploy, ide, monitor, db-client, cluster, deploy, tf-vae-test, deploy-job.
   path                                     The absolute or relative directory path where the script is located.
 
 Options:

package/manifests/deployment/{dd-template-development → dd-default-development}/deployment.yaml

@@ -2,22 +2,22 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: dd-template-development-blue
+  name: dd-default-development-blue
   labels:
-    app: dd-template-development-blue
+    app: dd-default-development-blue
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: dd-template-development-blue
+      app: dd-default-development-blue
   template:
     metadata:
       labels:
-        app: dd-template-development-blue
+        app: dd-default-development-blue
     spec:
       containers:
-        - name: dd-template-development-blue
-          image: localhost/rockylinux9-underpost:v2.8.852
+        - name: dd-default-development-blue
+          image: localhost/rockylinux9-underpost:v2.8.854
           #          resources:
           #            requests:
           #              memory: "124Ki"
@@ -40,10 +40,10 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: dd-template-development-blue-service
+  name: dd-default-development-blue-service
 spec:
   selector:
-    app: dd-template-development-blue
+    app: dd-default-development-blue
   ports:
     - name: 'tcp-4001'
       protocol: TCP
@@ -85,22 +85,22 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: dd-template-development-green
+  name: dd-default-development-green
   labels:
-    app: dd-template-development-green
+    app: dd-default-development-green
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: dd-template-development-green
+      app: dd-default-development-green
   template:
     metadata:
       labels:
-        app: dd-template-development-green
+        app: dd-default-development-green
     spec:
       containers:
-        - name: dd-template-development-green
-          image: localhost/rockylinux9-underpost:v2.8.852
+        - name: dd-default-development-green
+          image: localhost/rockylinux9-underpost:v2.8.854
           #          resources:
           #            requests:
           #              memory: "124Ki"
@@ -124,10 +124,10 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: dd-template-development-green-service
+  name: dd-default-development-green-service
 spec:
   selector:
-    app: dd-template-development-green
+    app: dd-default-development-green
   ports:
     - name: 'tcp-4001'
       protocol: TCP

package/manifests/deployment/{dd-template-development → dd-default-development}/proxy.yaml

@@ -16,7 +16,7 @@ spec:
         - prefix: /
       enableWebsockets: true
       services:
-        - name: dd-template-development-blue-service
+        - name: dd-default-development-blue-service
           port: 4001
           weight: 100
 
@@ -24,7 +24,7 @@ spec:
         - prefix: /peer
       enableWebsockets: true
       services:
-        - name: dd-template-development-blue-service
+        - name: dd-default-development-blue-service
           port: 4002
           weight: 100
 
@@ -41,6 +41,6 @@ spec:
         - prefix: /
       enableWebsockets: true
       services:
-        - name: dd-template-development-blue-service
+        - name: dd-default-development-blue-service
           port: 4003
           weight: 100

package/manifests/grafana/deployment.yaml

@@ -0,0 +1,57 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: grafana
+  name: grafana
+spec:
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      securityContext:
+        fsGroup: 472
+        supplementalGroups:
+          - 0
+      containers:
+        - name: grafana
+          image: grafana/grafana:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3000
+              name: http-grafana
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /robots.txt
+              port: 3000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 2
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3000
+            timeoutSeconds: 1
+          resources:
+            requests:
+              cpu: 250m
+              memory: 750Mi
+          volumeMounts:
+            - mountPath: /var/lib/grafana
+              name: grafana-pv
+      volumes:
+        - name: grafana-pv
+          persistentVolumeClaim:
+            claimName: grafana-pvc

package/manifests/grafana/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml

package/manifests/grafana/pvc.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 5Gi

package/manifests/grafana/service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+spec:
+  ports:
+    - port: 3000
+      protocol: TCP
+      targetPort: http-grafana
+  selector:
+    app: grafana
+  sessionAffinity: None
+  type: LoadBalancer

package/manifests/maas/ssh-cluster-info.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REMOTE_USER=$(node bin config get --plain DEFAULT_SSH_USER)
+REMOTE_HOST=$(node bin config get --plain DEFAULT_SSH_HOST)
+SSH_KEY=$(node bin config get --plain DEFAULT_SSH_KEY_PATH)
+
+chmod 600 "$SSH_KEY"
+
+ssh -i "$SSH_KEY" -o BatchMode=yes "${REMOTE_USER}@${REMOTE_HOST}" sh <<EOF
+cd /home/dd/engine
+node bin deploy dd production --info-traffic
+kubectl get pods -A
+EOF

package/manifests/prometheus/deployment.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  labels:
+    app: prometheus
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 30s
+      evaluation_interval: 30s
+
+    scrape_configs:
+      - job_name: 'scraper'
+        metrics_path: /metrics
+        scheme: https
+        static_configs:
+          - targets: []
+        # tls_config:
+        #   insecure_skip_verify: true
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      containers:
+        - name: prometheus
+          image: prom/prometheus:latest
+          args:
+            - '--config.file=/etc/prometheus/prometheus.yml'
+            - '--storage.tsdb.path=/prometheus'
+            - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+            - '--web.console.templates=/usr/share/prometheus/consoles'
+          ports:
+            - containerPort: 9090
+              name: web
+          volumeMounts:
+            - name: prometheus-config
+              mountPath: /etc/prometheus/
+            - name: prometheus-data
+              mountPath: /prometheus
+          resources:
+            requests:
+              memory: '200Mi'
+              cpu: '100m'
+      volumes:
+        - name: prometheus-config
+          configMap:
+            name: prometheus-config
+        - name: prometheus-data
+          emptyDir: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  labels:
+    app: prometheus
+spec:
+  ports:
+    - port: 9090
+      targetPort: 9090
+      protocol: TCP
+      name: web
+  selector:
+    app: prometheus
+  type: ClusterIP

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.852",
+    "version": "2.8.854",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -21,7 +21,6 @@
         "install-test": "npm install -g mocha && npm install -g c8 && npm install -g nyc && npm install -g coveralls",
         "install-underpost": "cp -a $(npm root -g)/underpost/node_modules ./node_modules && npm install --only=dev --ignore-scripts",
         "install": "npm run install-global && npm run install-test",
-        "docker:start": "docker-compose up",
         "prettier": "prettier --write .",
         "fix": "npm audit fix --force && npm audit",
         "changelog": "auto-changelog"

package/src/cli/cluster.js

@@ -5,6 +5,7 @@ import UnderpostBaremetal from './baremetal.js';
 import UnderpostDeploy from './deploy.js';
 import UnderpostTest from './test.js';
 import os from 'os';
+import fs from 'fs-extra';
 
 const logger = loggerFactory(import.meta);
 
@@ -33,12 +34,13 @@ class UnderpostCluster {
      * @param {string} [options.nsUse=''] - Set the current kubectl namespace.
      * @param {boolean} [options.infoCapacity=false] - Display resource capacity information for the cluster.
      * @param {boolean} [options.infoCapacityPod=false] - Display resource capacity information for pods.
-     * @param {boolean} [options.istio=false] - Deploy Istio service mesh.
      * @param {boolean} [options.pullImage=false] - Pull necessary Docker images before deployment.
      * @param {boolean} [options.dedicatedGpu=false] - Configure for dedicated GPU usage (e.g., NVIDIA GPU Operator).
      * @param {boolean} [options.kubeadm=false] - Initialize the cluster using Kubeadm.
      * @param {boolean} [options.k3s=false] - Initialize the cluster using K3s.
      * @param {boolean} [options.initHost=false] - Perform initial host setup (install Docker, Podman, Kind, Kubeadm, Helm).
+     * @param {boolean} [options.grafana=false] - Initialize the cluster with a Grafana deployment.
+     * @param {string} [options.prom=''] - Initialize the cluster with a Prometheus Operator deployment and monitor scrap for specified hosts.
      * @param {boolean} [options.uninstallHost=false] - Uninstall all host components.
      * @param {boolean} [options.config=false] - Apply general host configuration (SELinux, containerd, sysctl, firewalld).
      * @param {boolean} [options.worker=false] - Configure as a worker node (for Kubeadm or K3s join).
@@ -63,12 +65,13 @@ class UnderpostCluster {
         nsUse: '',
         infoCapacity: false,
         infoCapacityPod: false,
-        istio: false,
         pullImage: false,
         dedicatedGpu: false,
         kubeadm: false,
         k3s: false,
         initHost: false,
+        grafana: false,
+        prom: '',
         uninstallHost: false,
         config: false,
         worker: false,
@@ -260,6 +263,42 @@ class UnderpostCluster {
         );
       }
 
+      if (options.grafana === true) {
+        shellExec(`kubectl delete deployment grafana --ignore-not-found`);
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/grafana`);
+      }
+
+      if (options.prom && typeof options.prom === 'string') {
+        shellExec(`kubectl delete deployment prometheus --ignore-not-found`);
+        shellExec(`kubectl delete configmap prometheus-config --ignore-not-found`);
+        shellExec(`kubectl delete service prometheus --ignore-not-found`);
+        // Prometheus server host: http://<prometheus-cluster-ip>:9090
+        const yaml = `${fs.readFileSync(`${underpostRoot}/manifests/prometheus/deployment.yaml`, 'utf8').replace(
+          '- targets: []',
+          `- targets: [${options.prom
+            .split(',')
+            .map((host) => `'${host}'`)
+            .join(',')}]`,
+        )}`;
+        console.log(yaml);
+        shellExec(`kubectl apply -f - <<EOF
+${yaml}
+EOF
+`);
+
+        // https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/config-other-methods/prometheus/prometheus-operator/
+        // shellExec(
+        //   `kubectl create -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/master/bundle.yaml`,
+        // );
+        // shellExec(`kubectl apply -f ${underpostRoot}/manifests/prometheus/prometheus-cr.yaml`);
+        // shellExec(`kubectl rollout status sts prometheus-prometheus -n default`);
+        // shellExec(`kubectl apply -f ${underpostRoot}/manifests/prometheus/prometheus-server.yaml`);
+        // shellExec(`helm repo add prometheus-community https://prometheus-community.github.io/helm-charts`);
+        // shellExec(`helm repo update`);
+        // shellExec(`helm install prometheus prometheus-community/prometheus`);
+        // shellExec(`kubectl rollout status deployment prometheus-server -n default`);
+      }
+
       if (options.full === true || options.valkey === true) {
         if (options.pullImage === true) {
           // shellExec(`sudo podman pull valkey/valkey:latest`);

package/src/cli/cron.js

@@ -50,7 +50,7 @@ class UnderpostCron {
     callback: async function (
       deployList = 'default',
       jobList = Object.keys(UnderpostCron.JOB),
-      options = { itc: false, init: false, git: false, dashboardUpdate: false },
+      options = { itc: false, init: false, git: false },
     ) {
       if (options.init === true) {
         UnderpostCron.NETWORK = [];
@@ -61,7 +61,7 @@ class UnderpostCron {
           for (const job of Object.keys(confCronConfig.jobs)) {
             const name = `${jobDeployId}-${job}`;
             let deployId;
-            if (!options.dashboardUpdate) shellExec(Cmd.delete(name));
+            shellExec(Cmd.delete(name));
             switch (job) {
               case 'dns':
                 deployId = jobDeployId;
@@ -71,8 +71,7 @@ class UnderpostCron {
                 deployId = deployList;
                 break;
             }
-            if (!options.dashboardUpdate)
-              shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
             UnderpostCron.NETWORK.push({
               deployId,
               jobId: job,
@@ -80,7 +79,6 @@ class UnderpostCron {
             });
           }
         }
-        if (options.dashboardUpdate === true) await UnderpostCron.API.updateDashboardData();
         if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
         return;
       }
@@ -89,32 +87,6 @@ class UnderpostCron {
         if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
       }
     },
-    async updateDashboardData() {
-      try {
-        const deployId = process.env.DEFAULT_DEPLOY_ID;
-        const host = process.env.DEFAULT_DEPLOY_HOST;
-        const path = process.env.DEFAULT_DEPLOY_PATH;
-        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-        const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
-        const { db } = confServer[host][path];
-
-        await DataBaseProvider.load({ apis: ['cron'], host, path, db });
-
-        /** @type {import('../api/cron/cron.model.js').CronModel} */
-        const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
-
-        await Cron.deleteMany();
-
-        for (const cronInstance of UnderpostCron.NETWORK) {
-          logger.info('save', cronInstance);
-          await new Cron(cronInstance).save();
-        }
-
-        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
-      } catch (error) {
-        logger.error(error, error.stack);
-      }
-    },
   };
 }