samanbayaka 0.0.2

package/helper/file/esm-loading.mjs

@@ -0,0 +1,181 @@
+/*esm-loading.mjs*/
+
+import { fileURLToPath, pathToFileURL } from "url"
+import * as path from "path"
+import * as fs from "fs"
+import { createRequire } from 'module'
+
+import chokidar from "chokidar"
+
+import {isHotReloadEnabled, serviceName, nodeUid} from '#hUti/node-argv.mjs'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+const require = createRequire(import.meta.url)
+
+
+const CONFIG_PATH = process.env.SBK_CONFIG_PATH  // Environment variable of configuration path
+
+/** 
+ * Project absolute pat
+ */
+const ABSOLUTE_PATH = path
+    .join(
+      __dirname,
+      "..",   // back to parent directory
+      "..",   // back to parent directory
+    ) 
+
+/**
+ * Getting configurations through dynamic loading of ESM modules
+ * ftom the path defined in environment variable
+ * @param  {string}
+ * @return {object}
+ */
+export const getConfig = async (fileName) => {
+	const configFile = path.
+		join(
+			CONFIG_PATH,
+			`${fileName}.mjs`
+		)
+	return (await import(pathToFileURL(configFile).href)).default
+}
+
+
+/**
+ * Define Server configuration.
+ * @type {string}
+ */
+const CONFIG = await getConfig('server')
+
+
+/**
+ * Load modules from path as moleculer-repl is only supports cjs files
+ * @param  {string} directory path
+ * @param  {RegExp} regx to filter only *.mjs files from service path
+ * @return {void}
+ */
+export const loadServices = async (broker) => {
+  /**
+   * Derive the service directory from the server configuration.
+   * @type {string}
+   */
+	const SERVICES_DIR = path
+    .join(
+      ABSOLUTE_PATH,
+      CONFIG.servicesDir,
+      serviceName
+    ) 
+
+  const filePath = path.join(SERVICES_DIR, CONFIG.serviceMain)
+
+  /**
+   * Dynamic import with cache busting
+   * @type {object}
+   */
+  const module = await import(`${pathToFileURL(filePath)}?t=${Date.now()}`)
+  const schema = module.default || module
+
+  if (!schema || typeof schema !== "object" || !schema.name || schema.name != serviceName) {
+    const regex = new RegExp(`(\\/${CONFIG.servicesDir}\\/.*)$`)
+    const rfPath = pathToFileURL(filePath).href.match(regex)[1]
+    await broker.stop()
+    broker.logger.fatal('❖ SBK', `Service start failed: invalid service name or schema in ${rfPath}`)
+    process.exit(0)
+  }
+
+  broker.createService(schema)
+  broker.logger.info('❖ SBK', `Hot-realod: ${isHotReloadEnabled ? "ON" : "OFF"}`)
+  
+  const watcher = chokidar.watch(SERVICES_DIR, {
+    ignoreInitial: true,
+    awaitWriteFinish: {
+      stabilityThreshold: 200, // wait 200ms after last change
+      pollInterval: 50
+    },
+    ignored: [
+      /(^|[\/\\])\../,
+      /~$/,
+      /\.swp$/,
+      /\.tmp$/
+    ]
+  })
+
+  const handleWatchFile = async(wfPath, isUnlink) => {
+    const regex = new RegExp(`(\\/${CONFIG.servicesDir}\\/.*)$`)
+    const rfPath = pathToFileURL(wfPath).href.match(regex)[1]
+    /**
+     * Hot Reloading of moleculer services from service path
+     */
+    if(isHotReloadEnabled){
+      if(path.basename(filePath) === path.basename(wfPath)){
+        const oldServiceFullName = (broker.services.find(s => s.name === serviceName))?.fullName
+        const oldService = broker.getLocalService(oldServiceFullName)
+
+        if (!isUnlink) {
+          try {
+            /**
+             * Dynamic import with cache busting
+             */
+            const serviceModule = await import(`${pathToFileURL(filePath)}?t=${Date.now()}`)
+            const schema = serviceModule.default || serviceModule
+
+            if (!schema || typeof schema !== "object" || !schema.name || schema.name != serviceName) {
+              broker.logger.error('❖ SBK', `Hot-reloading terminated due to an invalid service name or schema for ${filePath}`)
+              return
+            }
+
+            /**
+             * Remove the already loaded service if it exists.
+             */
+            if (oldService) {
+              await broker.destroyService(oldService)
+            }
+
+            /**
+             * Create new service
+             */
+            broker.createService(schema)
+            const newServiceFullName = schema.version === undefined 
+              ? schema.name 
+              : `${schema.version}.${schema.name}`
+            broker.logger.info('❖ SBK', `Service '${newServiceFullName}' reloaded.`)
+          } catch (err) {
+            broker.logger.error('❖ SBK', `Hot-reloading of service '${newServiceFullName}' failed..`, err)
+          }
+        } else if (event === "unlink") {
+          /**
+           * Remove the already loaded service if it exists.
+           */
+          if (oldService) {
+            await broker.destroyService(oldService)
+            broker.logger.info('❖ SBK', `Service '${newServiceFullName}' removed.`) 
+          }
+        }
+      }
+      else {
+        broker.logger.warn('❖ SBK',`Changes detected in the file "${rfPath}". Restart required to apply updates.`)
+      }
+
+    }
+    else {
+      broker.logger.warn('❖ SBK',`Changes detected in the file "${rfPath}". Restart required to apply updates.`)
+    }
+  }
+
+  watcher.on("add", (fp) => handleWatchFile(fp, false))
+  watcher.on("change", (fp) => handleWatchFile(fp, false))
+  watcher.on("unlink", (fp) => handleWatchFile(fp, true))
+}
+
+
+/**
+ * Assets path
+ */
+export const assetPath = {
+  rootFolder: path
+    .join(
+      ABSOLUTE_PATH,
+      CONFIG.assetPath,
+    ),
+}

package/helper/transporter/RedpandaTransporter.mjs

@@ -0,0 +1,235 @@
+/*RedpandaTransporter.mjs*/
+import { Transporters } from "moleculer"
+import { Kafka, logLevel } from "kafkajs"
+
+/**
+ * Custom KafkaJS-backed Moleculer Transporter
+ *
+ * Topics used (prefix = MOL by default):
+ *  - <prefix>.<CMD>                          (fanout/control)
+ *  - <prefix>.<CMD>.<nodeID>                 (directed)
+ *
+ * Examples:
+ *  - MOL.DISCOVER, MOL.INFO, MOL.HEARTBEAT, MOL.PING, MOL.PONG
+ *  - MOL.REQ                                 (requests)
+ *  - MOL.RES.<nodeID>                        (responses back to caller node)
+ */
+export default class RedpandaTransporter extends Transporters.Base {
+  constructor(opts) {
+    super(opts)
+
+    this.opts = {
+      ...{
+        // visit for documentation of KafkaJs https://kafka.js.org/docs/getting-started
+
+        // KafkaJs Client options.
+        client: {
+          // A logical identifier of an application. Can be used by brokers to apply quotas or trace requests to a specific application. Example: booking-events-processor.
+          clientId: "moleculer-kafkajs",
+          brokers: ["redpanda1:9092", "redpanda2:9092"],
+
+          // // The ssl option can be used to configure the TLS sockets. The options are passed directly to tls.connect and used to create the TLS Secure Context, all options are accepted.
+          // ssl: {
+          //   rejectUnauthorized: false,
+          //   ca: [fs.readFileSync('/my/custom/ca.crt', 'utf-8')],
+          //   key: fs.readFileSync('/my/custom/client-key.pem', 'utf-8'),
+          //   cert: fs.readFileSync('/my/custom/client-cert.pem', 'utf-8')
+          // },
+
+          // // Kafka has support for using SASL to authenticate clients. The sasl option can be used to configure the authentication mechanism. Currently, KafkaJS supports PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, and AWS mechanisms.
+          // authenticationTimeout: 10000,
+          // reauthenticationThreshold: 10000,
+          // sasl: {
+          //   mechanism: 'plain', // scram-sha-256 or scram-sha-512
+          //   username: 'my-username',
+          //   password: 'my-password'
+          // },
+
+          // Time in milliseconds to wait for a successful connection. The default value is: 1000.
+          connectionTimeout: 3000,
+
+          // Time in milliseconds to wait for a successful request. The default value is: 30000.
+          requestTimeout: 25000,
+
+          // The request timeout can be disabled by setting enforceRequestTimeout to false.
+          enforceRequestTimeout: false,
+
+
+          // The retry option can be used to set the configuration of the retry mechanism, which is used to retry connections and API calls to Kafka (when using producers or consumers).
+          retry: {
+            initialRetryTime: 100,
+            retries: 8
+          },
+
+          // KafkaJS has a built-in STDOUT logger which outputs JSON. It also accepts a custom log creator which allows you to integrate your favorite logger library. There are 5 log levels available: NOTHING, ERROR, WARN, INFO, and DEBUG. INFO is configured by default.
+          logLevel: logLevel.INFO,
+        },
+
+        // KafkaJs Producer options.
+        producer: {
+          // If enabled producer will ensure each message is written exactly once. Acks must be set to -1 ("all"). Retries will default to MAX_SAFE_INTEGER.
+          idempotent: true,
+
+          // Max number of requests that may be in progress at any time. If falsey then no limit.
+          maxInFlightRequests: 1,
+        },
+        customPartitioner: undefined,
+
+        // KafkaJs ConsumerGroup options.
+        consumer: {
+          controlGroupId: null,
+          dataGroupId: null,
+        },
+
+        // Advanced options for `send`.
+        publish: {
+          partition: 0,
+          attributes: 0
+        },
+
+        prefix: "MOL",
+      },
+      ...opts
+    }
+
+    this.client = null
+    this.producer = null
+    this.consumer = null
+  }
+
+
+  /**
+   * Init transporter
+   */
+  init(...args) {
+    super.init(...args)
+    this.prefix = this.opts.prefix || "MOL"
+
+    this.opts.client.logLevel = 
+      typeof this.opts.client.logLevel == "string" 
+      ? logLevel[this.opts.client.logLevel]
+      : this.opts.client.logLevel
+
+    this.logger.info(`RedpandaTransporter initialized with topic prefix=${this.prefix}`)
+  }
+
+
+  /**
+   * Connect to Kafka broker
+   */
+  async connect() {
+    this.client = new Kafka(this.opts.client)
+
+    this.producer = this.client.producer()
+    await this.producer.connect()
+
+    // Consumer group is important for balancing
+    const groupId = this.opts.groupId || this.nodeID  //.split(".")[0]
+
+    this.consumer = this.client.consumer({ groupId })
+    await this.consumer.connect()
+
+    this.connected = true
+    this.logger.info("Kafka transporter connected.")
+
+    return this.onConnected(false)
+  }
+
+  /**
+   * Disconnect from Kafka
+   */
+  async disconnect() {
+    if (this.producer) await this.producer.disconnect()
+    if (this.consumer) await this.consumer.disconnect()
+    this.connected = false
+    this.logger.info("Kafka transporter disconnected.")
+  }
+
+  /**
+   * Subscribe to a topic (for non-balanced commands)
+   * @param {String} cmd
+   * @param {String} nodeID
+   */
+  async subscribe(cmd, nodeID) {
+    if (!this.connected) {
+      throw new BrokerDisconnectedError()
+    }
+
+    const topic = this.getTopicName(cmd, nodeID)
+
+    await this.consumer.subscribe({ topic, fromBeginning: false })
+
+    if (!this.consumerRunning) {
+      this.consumerRunning = true
+      // Single run handler for all topics
+      await this.consumer.run({
+        eachMessage: async ({ topic, message }) => {
+          try {
+            // Extract command from topic name
+            const parts = topic.split(".")
+            const cmd = parts[1] // e.g., INFO, HEARTBEAT, REQ, RES, EVENT      
+            this.receive(cmd, message.value)
+          } catch (err) {
+            this.logger.error("Kafka message handling failed", err)
+          }
+        },
+      })
+    }
+
+    this.logger.debug(`Subscribed to topic '${topic}'`)
+  }
+
+  /**
+   * Subscribe to balanced requests
+   * @param {String} action
+   */
+  async subscribeBalancedRequest(action) {
+    const topic = `${this.prefix}.REQB.${action}`
+    await this.consumer.subscribe({ topic, fromBeginning: false })
+
+    await this.consumer.run({
+      eachMessage: async ({ topic, message }) => {
+        this.receive("REQ", message.value)
+      },
+    })
+
+    this.logger.debug(`Subscribed to balanced request '${action}'`)
+  }
+
+  /**
+   * Subscribe to balanced events
+   * @param {String} event
+   * @param {String} group
+   */
+  async subscribeBalancedEvent(event, group) {
+    const topic = `${this.prefix}.EVTB.${group}.${event}`
+    await this.consumer.subscribe({ topic, fromBeginning: false })
+
+    await this.consumer.run({
+      eachMessage: async ({ topic, message }) => {
+        this.receive("EVENT", message.value)
+      },
+    })
+
+    this.logger.debug(`Subscribed to balanced event '${event}' (group: ${group})`)
+  }
+
+  /**
+   * Send a message to Kafka
+   * @param {String} topic
+   * @param {Buffer} data
+   * @param {Object} meta
+   */
+  async send(topic, data, meta) {
+    if (!this.connected) {
+      throw new BrokerDisconnectedError()
+    }
+    
+    await this.producer.send({
+      topic,
+      messages: [{ value: data }],
+    })
+
+    this.logger.debug(`Message published to '${topic}'`, meta?.packet?.type)
+  }
+}

package/helper/transporter/RedpandaTransporter.mjs.bk

@@ -0,0 +1,230 @@
+/*RedpandaTransporter.mjs*/
+import { Transporters } from "moleculer"
+import { Kafka, logLevel } from "kafkajs"
+
+/**
+ * Custom KafkaJS-backed Moleculer Transporter
+ *
+ * Topics used (prefix = MOL by default):
+ *  - <prefix>.<CMD>                          (fanout/control)
+ *  - <prefix>.<CMD>.<nodeID>                 (directed)
+ *
+ * Examples:
+ *  - MOL.DISCOVER, MOL.INFO, MOL.HEARTBEAT, MOL.PING, MOL.PONG
+ *  - MOL.REQ                                 (requests)
+ *  - MOL.RES.<nodeID>                        (responses back to caller node)
+ */
+export default class RedpandaTransporter extends Transporters.Base {
+  constructor(opts) {
+    super(opts)
+
+    this.opts = {
+      ...{
+        // visit for documentation of KafkaJs https://kafka.js.org/docs/getting-started
+
+        // KafkaJs Client options.
+        client: {
+          // A logical identifier of an application. Can be used by brokers to apply quotas or trace requests to a specific application. Example: booking-events-processor.
+          clientId: "moleculer-kafkajs",
+          brokers: ["redpanda1:9092", "redpanda2:9092"],
+
+          // // The ssl option can be used to configure the TLS sockets. The options are passed directly to tls.connect and used to create the TLS Secure Context, all options are accepted.
+          // ssl: {
+          //   rejectUnauthorized: false,
+          //   ca: [fs.readFileSync('/my/custom/ca.crt', 'utf-8')],
+          //   key: fs.readFileSync('/my/custom/client-key.pem', 'utf-8'),
+          //   cert: fs.readFileSync('/my/custom/client-cert.pem', 'utf-8')
+          // },
+
+          // // Kafka has support for using SASL to authenticate clients. The sasl option can be used to configure the authentication mechanism. Currently, KafkaJS supports PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, and AWS mechanisms.
+          // authenticationTimeout: 10000,
+          // reauthenticationThreshold: 10000,
+          // sasl: {
+          //   mechanism: 'plain', // scram-sha-256 or scram-sha-512
+          //   username: 'my-username',
+          //   password: 'my-password'
+          // },
+
+          // Time in milliseconds to wait for a successful connection. The default value is: 1000.
+          connectionTimeout: 3000,
+
+          // Time in milliseconds to wait for a successful request. The default value is: 30000.
+          requestTimeout: 25000,
+
+          // The request timeout can be disabled by setting enforceRequestTimeout to false.
+          enforceRequestTimeout: false,
+
+
+          // The retry option can be used to set the configuration of the retry mechanism, which is used to retry connections and API calls to Kafka (when using producers or consumers).
+          retry: {
+            initialRetryTime: 100,
+            retries: 8
+          },
+
+          // KafkaJS has a built-in STDOUT logger which outputs JSON. It also accepts a custom log creator which allows you to integrate your favorite logger library. There are 5 log levels available: NOTHING, ERROR, WARN, INFO, and DEBUG. INFO is configured by default.
+          logLevel: logLevel.INFO,
+        },
+
+        // KafkaJs Producer options.
+        producer: {
+          // If enabled producer will ensure each message is written exactly once. Acks must be set to -1 ("all"). Retries will default to MAX_SAFE_INTEGER.
+          idempotent: true,
+
+          // Max number of requests that may be in progress at any time. If falsey then no limit.
+          maxInFlightRequests: 1,
+        },
+        customPartitioner: undefined,
+
+        // KafkaJs ConsumerGroup options.
+        consumer: {
+          controlGroupId: null,
+          dataGroupId: null,
+        },
+
+        // Advanced options for `send`.
+        publish: {
+          partition: 0,
+          attributes: 0
+        },
+
+        prefix: "MOL",
+      },
+      ...opts
+    }
+
+    this.client = null
+    this.producer = null
+    this.consumer = null
+  }
+
+
+  /**
+   * Init transporter
+   */
+  init(...args) {
+    super.init(...args)
+    this.prefix = this.opts.prefix || "MOL"
+
+    this.opts.client.logLevel = 
+      typeof this.opts.client.logLevel == "string" 
+      ? logLevel[this.opts.client.logLevel]
+      : this.opts.client.logLevel
+
+    this.logger.info(`RedpandaTransporter initialized with topic prefix=${this.prefix}`)
+  }
+
+
+  /**
+   * Connect to Kafka broker
+   */
+  async connect() {
+    this.client = new Kafka(this.opts.client)
+
+    this.producer = this.client.producer()
+    await this.producer.connect()
+
+    // Consumer group is important for balancing
+    const groupId = this.opts.groupId || this.nodeID
+    this.consumer = this.client.consumer({ groupId })
+    await this.consumer.connect()
+
+    this.connected = true
+    this.logger.info("Kafka transporter connected.")
+
+    return this.onConnected(false)
+  }
+
+  /**
+   * Disconnect from Kafka
+   */
+  async disconnect() {
+    if (this.producer) await this.producer.disconnect()
+    if (this.consumer) await this.consumer.disconnect()
+    this.connected = false
+    this.logger.info("Kafka transporter disconnected.")
+  }
+
+  /**
+   * Subscribe to a topic (for non-balanced commands)
+   * @param {String} cmd
+   * @param {String} nodeID
+   */
+  async subscribe(cmd, nodeID) {
+    const topic = this.getTopicName(cmd, nodeID)
+
+    await this.consumer.subscribe({ topic, fromBeginning: false })
+
+    // Single run handler for all topics
+    await this.consumer.run({
+      eachMessage: async ({ topic, message }) => {
+        try {
+          // Extract command from topic name
+          const parts = topic.split(".")
+          const cmd = parts[1] // e.g., INFO, HEARTBEAT, REQ, RES, EVENT
+// console.log('CCCCCCCCCCCCCCCCCCCC', cmd, nodeID)          
+          this.receive(cmd, message.value)
+        } catch (err) {
+          this.logger.error("Kafka message handling failed", err)
+        }
+      },
+    })
+
+    this.logger.debug(`Subscribed to topic '${topic}'`)
+  }
+
+  /**
+   * Subscribe to balanced requests
+   * @param {String} action
+   */
+  async subscribeBalancedRequest(action) {
+    const topic = `${this.prefix}.REQB.${action}`
+    await this.consumer.subscribe({ topic, fromBeginning: false })
+
+    await this.consumer.run({
+      eachMessage: async ({ topic, message }) => {
+        this.receive("REQ", message.value)
+      },
+    })
+
+    this.logger.debug(`Subscribed to balanced request '${action}'`)
+  }
+
+  /**
+   * Subscribe to balanced events
+   * @param {String} event
+   * @param {String} group
+   */
+  async subscribeBalancedEvent(event, group) {
+    const topic = `${this.prefix}.EVTB.${group}.${event}`
+    await this.consumer.subscribe({ topic, fromBeginning: false })
+
+    await this.consumer.run({
+      eachMessage: async ({ topic, message }) => {
+        this.receive("EVENT", message.value)
+      },
+    })
+
+    this.logger.debug(`Subscribed to balanced event '${event}' (group: ${group})`)
+  }
+
+  /**
+   * Send a message to Kafka
+   * @param {String} topic
+   * @param {Buffer} data
+   * @param {Object} meta
+   */
+  async send(topic, data, meta) {
+    if (!this.connected) {
+      throw new BrokerDisconnectedError()
+    }
+    
+//     const kTopic = topic.split(".").slice(0,3).join(".")
+// console.log('PPPPPPPPPPPPPPPPPPPPPPPPPPPPP', kTopic, topic)
+    await this.producer.send({
+      topic,
+      messages: [{ value: data }],
+    })
+
+    this.logger.debug(`Message published to '${topic}'`, meta?.packet?.type)
+  }
+}

package/helper/transporter/rpk-commands.txt

@@ -0,0 +1,8 @@
+#check redpanda running status
+$rpk cluster info
+
+#view all topics
+$rpk --brokers redpanda:9092 topic list
+
+#delete all topics
+$rpk topic list --brokers redpanda:9092   | tail -n +2   | grep -v '^__'   | xargs -n1 rpk topic delete  --brokers redpanda:9092