underpost 2.8.64 → 2.8.67

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.64'
+      engine.version: '2.8.67'
     networks:
       - load-balancer
 

package/jsdoc.json

@@ -23,7 +23,7 @@
       "homepageTitle": "Docs",
       "favicon": "./public/www.nexodev.org/"
     },
-    "tutorials": "./public/www.nexodev.org/docs/learn"
+    "tutorials": "./public/www.nexodev.org/docs/references"
   },
   "markdown": {
     "hardwrap": false,

package/manifests/calico-custom-resources.yaml

@@ -0,0 +1,25 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+      - blockSize: 26
+        cidr: 192.168.0.0/16
+        encapsulation: VXLANCrossSubnet
+        natOutgoing: Enabled
+        nodeSelector: all()
+
+---
+# This section configures the Calico API server.
+# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}

package/manifests/deployment/adminer/deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: adminer
+  labels:
+    app: adminer
+    group: db
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: adminer
+  template:
+    metadata:
+      labels:
+        app: adminer
+        group: db
+    spec:
+      containers:
+        - name: adminer
+          image: adminer:4.7.6-standalone
+          ports:
+            - containerPort: 8080
+          env:
+            - name: ADMINER_DESIGN
+              value: pepa-linha
+            - name: ADMINER_DEFAULT_SERVER
+              value: postgres
+          resources:
+            limits:
+              memory: '256Mi'
+              cpu: '500m'

package/manifests/deployment/adminer/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+# kubectl apply -k manifests/deployment/adminer/.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml
+  - service.yaml

package/manifests/deployment/adminer/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: adminer
+  labels:
+    group: db
+spec:
+  type: ClusterIP
+  selector:
+    app: adminer
+  ports:
+    - port: 8080
+      targetPort: 8080

package/manifests/mongodb-4.4/service-deployment.yaml

@@ -16,7 +16,7 @@ spec:
       hostname: mongo
       containers:
         - name: mongodb
-          image: docker.io/library/mongo:4.4
+          image: mongo:4.4
           command: ['mongod', '--replSet', 'rs0', '--bind_ip_all']
           # -- bash
           # mongo

package/manifests/postgresql/configmap.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: postgres-config
+  labels:
+    app: postgres
+data:
+  POSTGRES_DB: postgresdb
+  POSTGRES_USER: admin

package/manifests/postgresql/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+# kubectl apply -k postgresql/.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pv.yaml
+  - pvc.yaml
+  - configmap.yaml
+  - statefulset.yaml
+  - service.yaml

package/manifests/postgresql/pv.yaml

@@ -0,0 +1,15 @@
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: postgres-pv-volume
+  labels:
+    type: local
+    app: postgres
+spec:
+  storageClassName: manual
+  capacity:
+    storage: 5Gi
+  accessModes:
+    - ReadWriteMany
+  hostPath:
+    path: '/mnt/data'

package/manifests/postgresql/pvc.yaml

@@ -0,0 +1,13 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: postgres-pv-claim
+  labels:
+    app: postgres
+spec:
+  storageClassName: manual
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi

package/manifests/postgresql/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres-service
+spec:
+  clusterIP: None
+  selector:
+    app: postgres
+  ports:
+    - port: 5432

package/manifests/postgresql/statefulset.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+spec:
+  serviceName: postgres
+  replicas: 3
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:latest
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 5432
+          envFrom:
+            - configMapRef:
+                name: postgres-config
+          env:
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-secret
+                  key: password
+          volumeMounts:
+            - mountPath: /var/lib/postgresql/data
+              name: postgredb
+      volumes:
+        - name: postgredb
+          persistentVolumeClaim:
+            claimName: postgres-pv-claim

package/manifests/valkey/statefulset.yaml

@@ -1,4 +1,3 @@
----
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -15,9 +14,14 @@ spec:
       labels:
         app: service-valkey
     spec:
+      # Prevent automatic token mounting if you're not using the default ServiceAccount
+      automountServiceAccountToken: false
+
       containers:
         - name: service-valkey
-          image: docker.io/valkey/valkey:latest
+          image: valkey/valkey:latest
+          # Ensure you pull only if not present (Never will error if missing)
+          imagePullPolicy: Never
           env:
             - name: TZ
               value: Europe/Zurich
@@ -35,5 +39,3 @@ spec:
             failureThreshold: 2
             periodSeconds: 30
             timeoutSeconds: 5
-      restartPolicy: Always
-      automountServiceAccountToken: false

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.64",
+    "version": "2.8.67",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -11,6 +11,7 @@
         "dev": "env-cmd -f .env.development node src/client.dev default",
         "dev-img": "env-cmd -f .env.development node src/server",
         "prod-img": "env-cmd -f .env.production node src/server",
+        "monitor": "pm2 start bin/deploy.js --name monitor -- monitor",
         "dev-api": "env-cmd -f .env.development nodemon --watch src --ignore src/client src/api",
         "dev-client": "env-cmd -f .env.development node src/client.dev",
         "proxy": "node src/proxy proxy",
@@ -69,7 +70,6 @@
         "cors": "^2.8.5",
         "d3": "^7.9.0",
         "dotenv": "^16.3.1",
-        "easy-json-schema": "^0.0.2-beta",
         "easymde": "^2.18.0",
         "env-cmd": "^10.1.0",
         "express": "^4.18.2",
@@ -119,13 +119,7 @@
         "vanilla-jsoneditor": "^2.3.2",
         "winston": "^3.11.0"
     },
-    "devDependencies": {
-        "clean-jsdoc-theme": "^4.3.0",
-        "easy-json-schema": "^0.0.2-beta",
-        "mocha": "^10.8.2",
-        "plantuml": "^0.0.2",
-        "swagger-autogen": "^2.23.7"
-    },
+    "devDependencies": {},
     "publishConfig": {
         "provenance": true,
         "access": "public",

package/src/api/user/user.service.js

@@ -225,8 +225,8 @@ const UserService = {
         } else throw new Error('invalid email or password');
 
       case 'guest': {
-        const user = await ValkeyAPI.valkeyObjectFactory('user', options);
-        await ValkeyAPI.setValkeyObject(user.email, user);
+        const user = await ValkeyAPI.valkeyObjectFactory(options, 'user');
+        await ValkeyAPI.setValkeyObject(options, user.email, user);
         return {
           token: hashJWT({ user: UserDto.auth.payload(user) }),
           user: selectDtoFactory(user, UserDto.select.get()),
@@ -325,15 +325,18 @@ const UserService = {
         return await User.find().select(UserDto.select.getAll());
 
       case 'auth': {
-        const user = (await ValkeyAPI.getValkeyObject(req.auth.user.email))
-          ? await ValkeyAPI.getValkeyObject(req.auth.user.email)
-          : await User.findOne({
-              _id: req.auth.user._id,
-            });
+        let user;
+        if (req.auth.user._id.match('guest')) {
+          user = await ValkeyAPI.getValkeyObject(options, req.auth.user.email);
+          if (!user) throw new Error('guest user expired');
+        } else
+          user = await User.findOne({
+            _id: req.auth.user._id,
+          });
 
         const file = await File.findOne({ _id: user.profileImageId });
 
-        if (!file && !(await ValkeyAPI.getValkeyObject(req.auth.user.email))) {
+        if (!file && !(await ValkeyAPI.getValkeyObject(options, req.auth.user.email))) {
           await User.findByIdAndUpdate(
             user._id,
             { profileImageId: await getDefaultProfileImageId(File) },
@@ -342,8 +345,8 @@ const UserService = {
             },
           );
         }
-        return (await ValkeyAPI.getValkeyObject(req.auth.user.email))
-          ? selectDtoFactory(await ValkeyAPI.getValkeyObject(req.auth.user.email), UserDto.select.get())
+        return (await ValkeyAPI.getValkeyObject(options, req.auth.user.email))
+          ? selectDtoFactory(await ValkeyAPI.getValkeyObject(options, req.auth.user.email), UserDto.select.get())
           : await User.findOne({
               _id: req.auth.user._id,
             }).select(UserDto.select.get());

package/src/cli/cluster.js

@@ -1,5 +1,4 @@
-import { timer } from '../client/components/core/CommonJs.js';
-import { cliSpinner, getNpmRootPath } from '../server/conf.js';
+import { getNpmRootPath } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import UnderpostDeploy from './deploy.js';
@@ -15,6 +14,7 @@ class UnderpostCluster {
         mongodb: false,
         mongodb4: false,
         mariadb: false,
+        postgresql: false,
         valkey: false,
         full: false,
         info: false,
@@ -23,10 +23,21 @@ class UnderpostCluster {
         reset: false,
         dev: false,
         nsUse: '',
+        infoCapacity: false,
+        infoCapacityPod: false,
+        istio: false,
+        pullImage: false,
       },
     ) {
+      // 1) Install kind, kubeadm, docker, podman
+      // 2) Check kubectl, kubelet, containerd.io
+      // 3) Install Nvidia drivers from Rocky Linux docs
+      // 4) Install LXD with MAAS from Rocky Linux docs
+      // 5) Install MAAS src from snap
       const npmRoot = getNpmRootPath();
       const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      if (options.infoCapacityPod === true) return logger.info('', UnderpostDeploy.API.resourcesFactory());
+      if (options.infoCapacity === true) return logger.info('', UnderpostCluster.API.getResourcesCapacity());
       if (options.reset === true) return await UnderpostCluster.API.reset();
       if (options.listPods === true) return console.table(UnderpostDeploy.API.get(podName ?? undefined));
 
@@ -63,26 +74,62 @@ class UnderpostCluster {
         shellExec(`kubectl get secrets --all-namespaces -o wide`);
         shellExec(`docker secret ls`);
         shellExec(`kubectl get crd --all-namespaces -o wide`);
+        shellExec(`sudo kubectl api-resources`);
         return;
       }
 
-      if (!UnderpostDeploy.API.get('kube-apiserver-kind-control-plane')[0]) {
+      if (
+        (!options.istio && !UnderpostDeploy.API.get('kube-apiserver-kind-control-plane')[0]) ||
+        (options.istio === true && !UnderpostDeploy.API.get('calico-kube-controllers')[0])
+      ) {
+        shellExec(`sudo setenforce 0`);
+        shellExec(`sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config`);
+        // sudo systemctl disable kubelet
+        // shellExec(`sudo systemctl enable --now kubelet`);
         shellExec(`containerd config default > /etc/containerd/config.toml`);
         shellExec(`sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
         // shellExec(`cp /etc/kubernetes/admin.conf ~/.kube/config`);
-        shellExec(`sudo systemctl restart kubelet`);
+        // shellExec(`sudo systemctl restart kubelet`);
         shellExec(`sudo service docker restart`);
         shellExec(`sudo systemctl enable --now containerd.service`);
-        shellExec(`sudo systemctl restart containerd`);
-        shellExec(
-          `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
-            options?.dev === true ? '-dev' : ''
-          }.yaml`,
-        );
-        shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+        shellExec(`sudo swapoff -a; sudo sed -i '/swap/d' /etc/fstab`);
+        if (options.istio === true) {
+          shellExec(`sysctl net.bridge.bridge-nf-call-iptables=1`);
+          shellExec(`sudo kubeadm init --pod-network-cidr=192.168.0.0/16`);
+          shellExec(`sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config`);
+          shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+          // https://docs.tigera.io/calico/latest/getting-started/kubernetes/quickstart
+          shellExec(
+            `sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/tigera-operator.yaml`,
+          );
+          // shellExec(
+          //   `wget https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/custom-resources.yaml`,
+          // );
+          shellExec(`sudo kubectl apply -f ./manifests/calico-custom-resources.yaml`);
+          shellExec(`sudo systemctl restart containerd`);
+        } else {
+          shellExec(`sudo systemctl restart containerd`);
+          shellExec(
+            `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
+              options?.dev === true ? '-dev' : ''
+            }.yaml`,
+          );
+          shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+        }
       } else logger.warn('Cluster already initialized');
 
       if (options.full === true || options.valkey === true) {
+        if (options.pullImage === true) {
+          // kubectl patch statefulset service-valkey --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value":"valkey/valkey:latest"}]'
+          // kubectl patch statefulset service-valkey -p '{"spec":{"template":{"spec":{"containers":[{"name":"service-valkey","imagePullPolicy":"Never"}]}}}}'
+          shellExec(`docker pull valkey/valkey`);
+          // shellExec(`sudo kind load docker-image valkey/valkey`);
+          // shellExec(`sudo podman pull docker.io/valkey/valkey:latest`);
+          // shellExec(`podman save -o valkey.tar valkey/valkey`);
+          // shellExec(`sudo kind load image-archive valkey.tar`);
+          // shellExec(`sudo rm -rf ./valkey.tar`);
+          shellExec(`sudo kind load docker-image valkey/valkey:latest`);
+        }
         shellExec(`kubectl delete statefulset service-valkey`);
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/valkey`);
       }
@@ -96,7 +143,17 @@ class UnderpostCluster {
         shellExec(`kubectl delete statefulset mariadb-statefulset`);
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb`);
       }
+      if (options.full === true || options.postgresql === true) {
+        shellExec(
+          `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password`,
+        );
+        shellExec(`kubectl apply -k ./manifests/postgresql`);
+      }
       if (options.mongodb4 === true) {
+        if (options.pullImage === true) {
+          shellExec(`docker pull mongo:4.4`);
+          shellExec(`sudo kind load docker-image mongo:4.4`);
+        }
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb-4.4`);
 
         const deploymentName = 'mongodb-deployment';
@@ -196,6 +253,51 @@ class UnderpostCluster {
         `sudo sed -i -e "s@/var/lib/containers/storage@/home/containers/storage@g" /etc/containers/storage.conf`,
       );
       shellExec(`sudo podman system reset -f`);
+      // https://github.com/kubernetes-sigs/kind/issues/2886
+      shellExec(`sysctl net.bridge.bridge-nf-call-iptables=0`);
+      shellExec(`sysctl net.bridge.bridge-nf-call-arptables=0`);
+      shellExec(`sysctl net.bridge.bridge-nf-call-ip6tables=0`);
+      shellExec(`docker network rm kind`);
+    },
+    getResourcesCapacity() {
+      const resources = {};
+      const info = true
+        ? `Capacity:
+  cpu:                8
+  ephemeral-storage:  153131976Ki
+  hugepages-1Gi:      0
+  hugepages-2Mi:      0
+  memory:             11914720Ki
+  pods:               110
+Allocatable:
+  cpu:                8
+  ephemeral-storage:  153131976Ki
+  hugepages-1Gi:      0
+  hugepages-2Mi:      0
+  memory:             11914720Ki
+  pods: `
+        : shellExec(`kubectl describe node kind-worker | grep -E '(Allocatable:|Capacity:)' -A 6`, {
+            stdout: true,
+            silent: true,
+          });
+      info
+        .split('Allocatable:')[1]
+        .split('\n')
+        .filter((row) => row.match('cpu') || row.match('memory'))
+        .map((row) => {
+          if (row.match('cpu'))
+            resources.cpu = {
+              value: parseInt(row.split(':')[1].trim()) * 1000,
+              unit: 'm',
+            };
+          if (row.match('memory'))
+            resources.memory = {
+              value: parseInt(row.split(':')[1].split('Ki')[0].trim()),
+              unit: 'Ki',
+            };
+        });
+
+      return resources;
     },
   };
 }

package/src/cli/db.js

@@ -15,11 +15,13 @@ class UnderpostDB {
         export: false,
         podName: false,
         ns: false,
-        collection: '',
+        collections: '',
         outPath: '',
         drop: false,
         preserveUUID: false,
         git: false,
+        hosts: '',
+        paths: '',
       },
     ) {
       const newBackupTimestamp = new Date().getTime();
@@ -39,20 +41,28 @@ class UnderpostDB {
               if (!dbs[provider]) dbs[provider] = {};
 
               if (!(name in dbs[provider]))
-                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-') };
+                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-'), host, path };
             }
           }
         }
 
-        if (!fs.existsSync(`../${repoName}`)) {
-          shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
-        } else {
-          shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+        if (options.git === true) {
+          if (!fs.existsSync(`../${repoName}`)) {
+            shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
+          } else {
+            shellExec(`cd ../${repoName} && git checkout . && git clean -f -d`);
+            shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+          }
         }
 
         for (const provider of Object.keys(dbs)) {
           for (const dbName of Object.keys(dbs[provider])) {
-            const { hostFolder, user, password } = dbs[provider][dbName];
+            const { hostFolder, user, password, host, path } = dbs[provider][dbName];
+            if (
+              (options.hosts && !options.hosts.split(',').includes(host)) ||
+              (options.paths && !options.paths.split(',').includes(path))
+            )
+              continue;
             if (hostFolder) {
               logger.info('', { hostFolder, provider, dbName });
 
@@ -153,7 +163,7 @@ class UnderpostDB {
                       const podName = podNameData.NAME;
                       shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
                       if (options.collections)
-                        for (const collection of options.collections)
+                        for (const collection of options.collections.split(','))
                           shellExec(
                             `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /"`,
                           );