samanbayaka 0.0.26 → 0.0.27

package/README.md

@@ -436,6 +436,8 @@ SBK_CONFIG_CRED='<your_etcs_user> <your_etcd_pass> <your_etcd_port>' node index.
 ```
 or
 ```bash
+rm -rf node_modules/samanbayaka/ &&cp -r ../../samanbayaka/. node_modules/samanbayaka/ && SBK_CONFIG_CRED='sbk sbk@123 12379' node index.mjs
+
 pm2 stop "$(basename "$PWD")" && rm -rf node_modules/samanbayaka/ &&cp -r ../../samanbayaka/. node_modules/samanbayaka/ && SBK_CONFIG_CRED='sbk sbk@123 12379' pm2 start index.mjs --name "$(basename "$PWD")" --stop-exit-codes 1 --output /var/log/samanbayaka/sbk-out-${HOSTNAME}-$$.log --error /var/log/samanbayaka/sbk-err-${HOSTNAME}-$$.log
 ``` 
  * production
@@ -618,6 +620,8 @@ services:
         etcdctl --endpoints=http://etcd:2379 --user=root:${ETCD_ROOT_PW} user grant-role msg msg
 
         # ---------------------------------------------------------
+        etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/sys/yaml/SYS_CONFIGS "$(cat /etcd-data/sysconfig.yml)" || true
+
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/global/envs/REVISION "" || true
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/global/envs/APP_ENV development || true
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/global/envs/LOG_LEVEL warn || true
@@ -652,14 +656,12 @@ services:
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/demo true || true
         
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/system true || true
-        etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/system/envs/PORT 8765 || true
-        etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/system/envs/BROADCAST_INTERVAL 600000 || true
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/system/envs/LOG_LEVEL error || true
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/system/envs/MAX_L1_TTL 0 || true
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/system/envs/MAX_L2_TTL 0 || true
 
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/gateway true || true
-        etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/gateway/envs/PORT 9876 || true
+        etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/gateway/envs/PORT 8765 || true
 
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/test true || true
         etcdctl --endpoints=http://etcd:2379 --user=configadmin:${ETCD_CONFIG_ADMIN_PW} put /config/sbk/members/sms add || true

package/commit-hash.mjs

@@ -1 +1 @@
-export const COMMIT_HASH = '2a7a879';
+export const COMMIT_HASH = '64e1dcc';

package/config/nats.yml

@@ -10,7 +10,6 @@ retryPolicy:
   delay: 200
   maxDelay: 3000
   factor: 2
-  check: null # err => err && !!err.retryable
 
 contextParamsCloning: false
 

package/config/sysconfigs.yml

@@ -0,0 +1,24 @@
+# Expose the system API on the specified port
+port: <your_sys_port>
+
+# Project description
+projDescription: "{#var#} is a microservices-based project built with moleculer and moleculer-web, designed for scalable, high-performance service communication and API management."
+
+# Configuration brodcast interval in miliseconds
+broadcastInterval: 600000  # 10 minitues
+
+# The broadcast interval increases by the specified factor after each broadcast,
+# i.e., interval = factor × interval.
+factor: 1.1
+
+# Configuration broadcast maximum duration in miliseconds
+maxBroadcastDuration: 86400000  # 24 hours
+
+# Expose the API to the domain
+domain: sbk.wbsedcl.in
+
+# Directory Path for API Assets
+assetPath: public
+
+# URL of Redpanda web client
+redpandaWebUiPath: http://redpanda.web.ui:9080

package/helper/file/esm-loading.mjs

@@ -7,24 +7,11 @@ import os from "os"
 import { createRequire } from 'module'
 
 import chokidar from "chokidar"
-import * as configHld from '#hUti/config-handler.mjs'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
 const require = createRequire(import.meta.url)
 
-/**
- * Edge configs
- */
-const EC_ASSET_PATH = (await configHld.getConfigs(`/config/sbk/edge/asset/path`)) || "public"
-
-
-/**
- * Configuration path
- * @type {string}
- */
-const CONFIG_PATH = process.env.SBK_CONFIG_PATH  // Environment variable of configuration path
-
 
 /** 
  * Project absolute path
@@ -50,12 +37,12 @@ const SERVICES_DIR = /node_modules/.test(__dirname)
 /**
  * Assets path
  */
-export const assetPath = {
-  rootFolder: path
+export const absoluteAssetPath = (dirName) => {
+  return path
     .join(
       ABSOLUTE_PATH,
-      EC_ASSET_PATH,
-    ),
+      dirName,
+    )
 }
 
 
@@ -103,7 +90,7 @@ const pkgMain = readPkg(ABSOLUTE_PATH)
  * Create and object of name and version
  * @type {Object}
  */
-export const pkgDtls = {
+export const pkgInfo = {
   name: pkgMain.name.split('-')[0]
   .split(" ")
   .map(word => word.charAt(0).toUpperCase() + word.slice(1))
@@ -113,139 +100,22 @@ export const pkgDtls = {
 }
 
 
-// /**
-//  * OpenAPI configuration
-//  */
-// export const openApiConfig = {
-//   info: {
-//     title: pkgDtls.name,
-//     version: pkgDtls.version,
-//     summary: CONFIG.openApiInfo[0].summary,
-//     description: CONFIG.openApiInfo[0].description,
-//   },
-//   servers: CONFIG.openApiInfo[0].url,
-// }
-
-// /**
-//  * Load modules from path as moleculer-repl is only supports cjs files
-//  * @param  {string} directory path
-//  * @param  {RegExp} regx to filter only *.mjs files from service path
-//  * @return {void}
-//  */
-// export const loadServices = async (broker) => {
-//   /**
-//    * Derive the service directory from the server configuration.
-//    * @type {string}
-//    */
-// 	const SERVICES_DIR = path
-//     .join(
-//       ABSOLUTE_PATH,
-//       CONFIG.servicesDir,
-//       serviceName
-//     ) 
-
-//   const filePath = path.join(SERVICES_DIR, CONFIG.serviceMain)
-
-//   /**
-//    * Dynamic import with cache busting
-//    * @type {object}
-//    */
-//   const module = await import(`${pathToFileURL(filePath)}?t=${Date.now()}`)
-//   const schema = module.default || module
-
-//   if (!schema || typeof schema !== "object" || !schema.name || schema.name != serviceName) {
-//     const regex = new RegExp(`(\\/${CONFIG.servicesDir}\\/.*)$`)
-//     const rfPath = pathToFileURL(filePath).href.match(regex)[1]
-//     await broker.stop()
-//     broker.logger.fatal('SBKERR ❖', `Service start failed: invalid service name or schema in ${rfPath}`)
-//     process.exit(0)
-//   }
-
-//   broker.createService(schema)
-//   broker.logger.info('SBKMSG ❖', `Hot-realod: ${isHotReloadEnabled ? "ON" : "OFF"}`)
-  
-//   const watcher = chokidar.watch(SERVICES_DIR, {
-//     ignoreInitial: true,
-//     awaitWriteFinish: {
-//       stabilityThreshold: 200, // wait 200ms after last change
-//       pollInterval: 50
-//     },
-//     ignored: [
-//       /(^|[\/\\])\../,
-//       /~$/,
-//       /\.swp$/,
-//       /\.tmp$/
-//     ]
-//   })
-
-//   const handleWatchFile = async(wfPath, isUnlink) => {
-//     const regex = new RegExp(`(\\/${CONFIG.servicesDir}\\/.*)$`)
-//     const rfPath = pathToFileURL(wfPath).href.match(regex)[1]
-//     /**
-//      * Hot Reloading of moleculer services from service path
-//      */
-//     if(isHotReloadEnabled){
-//       if(path.basename(filePath) === path.basename(wfPath)){
-//         const oldServiceFullName = (broker.services.find(s => s.name === serviceName))?.fullName
-//         const oldService = broker.getLocalService(oldServiceFullName)
-
-//         if (!isUnlink) {
-//           try {
-//             /**
-//              * Dynamic import with cache busting
-//              */
-//             const serviceModule = await import(`${pathToFileURL(filePath)}?t=${Date.now()}`)
-//             const schema = serviceModule.default || serviceModule
-
-//             if (!schema || typeof schema !== "object" || !schema.name || schema.name != serviceName) {
-//               broker.logger.error('SBKERR ❖', `Hot-reloading terminated due to an invalid service name or schema for ${filePath}`)
-//               return
-//             }
-
-//             /**
-//              * Remove the already loaded service if it exists.
-//              */
-//             if (oldService) {
-//               await broker.destroyService(oldService)
-//             }
-
-//             /**
-//              * Create new service
-//              */
-//             broker.createService(schema)
-//             const newServiceFullName = schema.version === undefined 
-//               ? schema.name 
-//               : `${schema.version}.${schema.name}`
-//             broker.logger.info('SBKMSG ❖', `Service '${newServiceFullName}' reloaded.`)
-//           } catch (err) {
-//             broker.logger.error('SBKERR ❖', `Hot-reloading of service '${newServiceFullName}' failed..`, err)
-//           }
-//         } else if (event === "unlink") {
-//           /**
-//            * Remove the already loaded service if it exists.
-//            */
-//           if (oldService) {
-//             await broker.destroyService(oldService)
-//             broker.logger.info('SBKMSG ❖', `Service '${newServiceFullName}' removed.`) 
-//           }
-//         }
-//       }
-//       else {
-//         broker.logger.warn('SBKWRN ❖',`Changes detected in the file "${rfPath}". Restart required to apply updates.`)
-//       }
-
-//     }
-//     else {
-//       broker.logger.warn('SBKWRN ❖',`Changes detected in the file "${rfPath}". Restart required to apply updates.`)
-//     }
-//   }
-
-//   watcher.on("add", (fp) => handleWatchFile(fp, false))
-//   watcher.on("change", (fp) => handleWatchFile(fp, false))
-//   watcher.on("unlink", (fp) => handleWatchFile(fp, true))
-// }
-
-
+/**
+ * File chage watcher in the service directory
+ */
+export const dirWatcher = chokidar.watch(SERVICES_DIR, {
+  ignoreInitial: true,
+  awaitWriteFinish: {
+    stabilityThreshold: 2000, // wait 2000ms after last change
+    pollInterval: 50
+  },
+  ignored: [
+    /(^|[\/\\])\../,
+    /~$/,
+    /\.swp$/,
+    /\.tmp$/
+  ]
+})
 
 
 

package/helper/mol-built-in/AjvValidator.mjs

@@ -68,19 +68,6 @@ const formatAjvErrors = (errors, params) => {
     	path,
     	message: `The value '${val}' ${err.message}`
     }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
   })
+  
 }

package/helper/utility/config-handler.mjs

@@ -24,19 +24,18 @@ export const client = new Etcd3({
 })
 
 /**
- * Getting configurations through dynamic loading of ESM modules
- * ftom the path defined in environment variable
+ * Getting configurations against a Etcd key
  * @param  {string}
- * @return {object}
+ * @return {object | string}
  */
-export const getConfigs = async (path) => {
+export const getConfigs = async (path, isYaml = false) => {
   try {
-    const yamlText = await client.get(path)
+    const yamlText = (await client.get(path))?.toString()
     if ( path.split('/').includes('yaml') ) {
-      return Object.freeze( YAML.parse(yamlText?.toString()) )
+      return isYaml ? yamlText : Object.freeze( YAML.parse( yamlText ) )
     }
     else {
-      return yamlText?.toString()
+      return yamlText
     }
     
   } catch (err) {
@@ -44,7 +43,12 @@ export const getConfigs = async (path) => {
   }
 }
 
-
+/**
+ * Getting all keys having prefix path
+ * @param  {string}  path  
+ * @param  {Boolean} fullPath
+ * @return {array}
+ */
 export const getConfigKeys = async (path, fullPath = false) => {
   try {
     const entries = await client.getAll()
@@ -60,6 +64,35 @@ export const getConfigKeys = async (path, fullPath = false) => {
   }
 }
 
+/**
+ * Create Etcd key with value
+ * @param  {string} path
+ * @param  {string} data
+ */
+export const putConfig = async(path, data) => {
+  await client.put(path)
+  .value(data)
+}
+
+/**
+ * Delete Etcd key
+ * @param  {string} path
+ * @return {string}
+ */
+export const delConfig = async(path) => {
+  const deletedData = await client
+    .delete()
+    .key(path)
+    .getPrevious()
+
+  return deletedData.map(item => ({
+    ...item,
+    key: Buffer.from(item.key).toString("utf8"),
+    value: Buffer.from(item.value).toString("utf8"),
+  }))
+  
+}
+
 
 export const getConfigAll = async (path, fullPath = false) => {
   try {
@@ -115,8 +148,5 @@ export const watchConfigs = async (path, callback) => {
 
 }
 
-export const putConfig = async(path, data) => {
-  await client.put(path)
-  .value(data)
-}
+
 

package/helper/utility/error-handler.mjs

@@ -250,7 +250,7 @@ export const ErrorFormatterMW = {
         return await next(ctx)
       } 
       catch (err) {
-        if(!err?.data?.traceId){
+        if(!err?.data?.traceId) {          
           const requestID = ctx?.requestID
           const status = err?.code || 500
           const type = err?.type || 'INTERNAL_ERROR'
@@ -301,7 +301,7 @@ export const ErrorFormatterMW = {
  * Ensures all errors include a standard set of fields such as status code,
  * message, and optional metadata for client consumption.
  */
-export const httpErrorFormatter = (req, res, err)=>{  
+export const httpErrorFormatter = (req, res, err)=> { 
   const code = err.code || 500
   const errorBody = {
     ...(err?.data || { message: err.message }),

package/helper/utility/global-configs-validator.mjs

@@ -4,13 +4,13 @@ export const validateGlobalConfigs = async(prefix) => {
   try{
     const nodeVer = errHdl.checkMinNoveVer()
     const configHdl = await import("./config-handler.mjs")
-    const { serviceDtls } = await import("#hFil/esm-loading.mjs")
+    const { serviceDtls, dirWatcher } = await import("#hFil/esm-loading.mjs")
     const globalConfigs = await configHdl.getConfigAll(prefix)
 
 
     const isServiceRemoved = (((await configHdl.getConfigs( `/config/sbk/members/${serviceDtls.name}` )) || "false") == "false")
     if ( serviceDtls.name != "samanbayaka" && isServiceRemoved ) {
-      throw new Error (`The service "${serviceDtls.name}" is not in the trusted list. Please contact your CONFIGADMIN for assistance.`)
+      throw new Error (`The service "${serviceDtls.name}" is disallowed by CONFIGADMIN.`)
     }
 
     return Object.freeze({
@@ -22,6 +22,7 @@ export const validateGlobalConfigs = async(prefix) => {
       telemetry: errHdl.validateTelemetry(globalConfigs?.TELEMETRY),
       configRevision: errHdl.validateConfRevision(globalConfigs?.REVISION),
       serviceDtls,
+      dirWatcher,
       hostEntry: {
         nats: await errHdl.validateHostEntry('nats'),
         redis: await errHdl.validateHostEntry('redis'),

package/helper/utility/openapi-to-mol-params.mjs

@@ -0,0 +1,71 @@
+/**
+ * Generate Moleculer params definitions from the OpenAPI schema to enable request parameter validation using AJV
+ * @param  {object} schema
+ */
+export const openApiToMoleculerParams = (schema) => {
+  if( schema?.openapi?.parameters || schema?.openapi?.requestBody ){
+    schema.params = {
+      type: "object",
+      properties: {
+        params: {
+          type: "object",
+          properties: {
+          },
+          required: [],
+          additionalProperties: false,
+        },
+        query: {
+          type: "object",
+          properties: {
+
+          },
+          required: [],
+        },
+        body: {}
+      },
+      required:[],
+      additionalProperties: false,
+    }
+          
+          
+    if( schema?.openapi?.requestBody?.required ){
+      schema.params.required.push("body")
+    }
+
+    if( schema?.openapi?.requestBody  && schema?.openapi?.requestBody?.content ){
+      schema.params.properties.body.anyOf = Object.entries(schema?.openapi?.requestBody?.content)
+        .map(el=>el[1].schema)
+
+    }
+
+    if( schema?.openapi?.parameters ) {
+      schema.params.properties.params.properties = schema?.openapi?.parameters
+        .filter(el=> el.in == "path")
+        .reduce((obj, item) => {
+          obj[item.name] = item.schema
+          return obj
+        }, {})
+      schema.params.properties.params.required = Object.keys(schema.params.properties.params.properties)
+
+      if( schema.params.properties.params.required.length > 0 ){
+        schema.params.required.push("params")
+      }
+
+      schema.params.properties.query.properties = schema?.openapi?.parameters
+      .filter(el=> el.in == "query").reduce((obj, item) => {
+        obj[item.name] = item.schema
+        return obj
+      }, {})
+
+      if( Object.keys(schema.params.properties.query.properties).length > 0 ){
+        schema.params.required.push("query")
+      }
+
+      schema.params.properties.query.required = schema?.openapi?.parameters
+      .filter(el=> el.in == "query")
+      .filter(el => el.required)
+      .map(el => el.name )
+    }
+  }
+
+}

package/index.mjs

@@ -10,6 +10,7 @@ import HybridCacher  from "#hMol/HybridCacher.mjs"
 
 import { gracefulShutdown, validateServiceName, ErrorFormatterMW } from '#hUti/error-handler.mjs'
 import { createOpenTelemetryExporters, OpenTelemetryMW } from '#hUti/telemetry.mjs'
+import { openApiToMoleculerParams } from '#hUti/openapi-to-mol-params.mjs'
 
 /**
  * Global Configs
@@ -137,6 +138,18 @@ const createSbkServices = async(schemas, isREPL = false) => {
       ...configRevisionListner
     }
 
+    /**
+     * Override params according to the OpenAPI schema definition
+     */
+    if( schema?.actions ) {
+      Object.keys(schema.actions).map((el) => {
+        if( schema.actions[el] ){
+          openApiToMoleculerParams(schema.actions[el])
+
+        }
+      })
+    }
+    
     if( index == 0 ) validateServiceName(schema, SBK_GLOBAL_CONFIGS.serviceDtls.name)
     broker.createService(schema)
   }
@@ -156,6 +169,17 @@ const createSbkServices = async(schemas, isREPL = false) => {
   }
 
 
+  /**
+   * Notify on changes to files in the service directory.
+   */
+  SBK_GLOBAL_CONFIGS.dirWatcher.on("add", (fp) => {
+    broker.logger.warn({message: `The new file "${fp}" has been added to your service folder. Restart your service for the changes to take effect.`})
+  })
+  SBK_GLOBAL_CONFIGS.dirWatcher.on("change", (fp) => {
+    broker.logger.warn({message: `The file "${fp}" in your service folder has been modified. Restart your service for the changes to take effect.`})
+  })
+
+
   /**
    * Listen for force stop signals
    */
@@ -194,7 +218,6 @@ export default {
    * Loading Demo
    */
   createDemo: async() => {
-    const { default: about } = await import('#sAbt/index.mjs')
     const { default: apiGateway } = await import('#sApi/index.mjs')
     const demo = await import('#sDmo/index.mjs')
 
@@ -207,7 +230,6 @@ export default {
     })
     createSbkServices([
       gtWy,
-      about, 
       demo.user, 
       demo.role, 
       demo.mail, 
@@ -227,9 +249,8 @@ export default {
   },
 
   registerEdgeAPI: async () => {
-    const { default: about } = await import('#sAbt/index.mjs')
     const { default: apiGateway } = await import('#sApi/index.mjs')
-    createSbkServices([apiGateway, about])
+    createSbkServices(apiGateway)
 
   },
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "samanbayaka",
-  "version": "0.0.26",
+  "version": "0.0.27",
   "description": "Moleculer Gateway service with kafka transporter",
   "homepage": "https://gitlab.com/dalal.suvendu/samanbayaka#readme",
   "bugs": {
@@ -40,6 +40,7 @@
     "@opentelemetry/semantic-conventions": "^1.40.0",
     "ajv": "^8.18.0",
     "approve-builds": "^1.0.0",
+    "body-parser": "^2.3.0",
     "chokidar": "^5.0.0",
     "compression": "^1.8.1",
     "cookie-parser": "^1.4.7",

package/public/config-manager.html

@@ -1,4 +1,3 @@
-```html
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -624,4 +623,3 @@ Promise.all([
 
 </body>
 </html>
-```

package/public/scalar/index.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Scalar API Reference</title>
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1" />
+  </head>
+
+  <body>
+    <div id="app"></div>
+
+    <!-- Load the Script -->
+    <script src="./js/standalone.js"></script>
+
+    <!-- Initialize the Scalar API Reference -->
+    <script>
+      Scalar.createApiReference('#app', {
+        // The URL of the OpenAPI/Swagger document
+        url: '../sbk/system/docs/openapi/json',
+        // Avoid CORS issues
+        // proxyUrl: 'https://proxy.scalar.com',
+      })
+    </script>
+  </body>
+</html>

package/services/about/index.mjs

@@ -21,7 +21,8 @@ export default {
       },
 
       handler: async(ctx) => {
-        return `Welcome to ${pkgDtls.name} is a microservices-based project built with moleculer and moleculer-web, designed for scalable, high-performance service communication and API management.`
+        return ctx.call("system.projDesc")
+        // return `${pkgDtls.name} is a microservices-based project built with moleculer and moleculer-web, designed for scalable, high-performance service communication and API management.`
       },
     },
     version: {