wsp-ms-core 1.1.23 → 1.1.26

package/dist/index.d.mts

@@ -640,12 +640,29 @@ declare class HttpNotFoundController implements HttpController {
 }
 
 declare class KafkaManager extends EventManager {
-    private readonly kafka;
-    private readonly consumer;
-    private readonly producer;
+    private readonly runtimeProcess;
+    private kafka;
+    private consumer;
+    private producer;
+    private brokers;
+    private readonly seedBrokers;
+    private readonly groupId;
+    private leaderSelectionPromise;
     private producerConnected;
+    private isRecoveringLeader;
     constructor(connection: EventManagerConnection);
     private ensureProducerConnected;
+    private initializeKafkaClients;
+    private pinToLeaderBroker;
+    private buildKafkaClient;
+    private fetchLeaderBroker;
+    private recoverFromLeaderChange;
+    private waitForLeaderSelection;
+    private resolveBrokerList;
+    private normalizeBroker;
+    private registerConsumerEventHandlers;
+    private logConsumerEvent;
+    private handleKafkaError;
     private commitMessage;
     private run;
     send(e: EventMessage): Promise<void>;

package/dist/index.d.ts

@@ -640,12 +640,29 @@ declare class HttpNotFoundController implements HttpController {
 }
 
 declare class KafkaManager extends EventManager {
-    private readonly kafka;
-    private readonly consumer;
-    private readonly producer;
+    private readonly runtimeProcess;
+    private kafka;
+    private consumer;
+    private producer;
+    private brokers;
+    private readonly seedBrokers;
+    private readonly groupId;
+    private leaderSelectionPromise;
     private producerConnected;
+    private isRecoveringLeader;
     constructor(connection: EventManagerConnection);
     private ensureProducerConnected;
+    private initializeKafkaClients;
+    private pinToLeaderBroker;
+    private buildKafkaClient;
+    private fetchLeaderBroker;
+    private recoverFromLeaderChange;
+    private waitForLeaderSelection;
+    private resolveBrokerList;
+    private normalizeBroker;
+    private registerConsumerEventHandlers;
+    private logConsumerEvent;
+    private handleKafkaError;
     private commitMessage;
     private run;
     send(e: EventMessage): Promise<void>;

package/dist/index.js

@@ -1698,13 +1698,33 @@ var HttpNotFoundController = class {
 };
 
 // src/infrastructure/kafka/KafkaManager.ts
-import { Kafka, logLevel } from "kafkajs";
+import {
+  Kafka,
+  KafkaJSRequestTimeoutError,
+  logLevel
+} from "kafkajs";
 var KafkaManager = class extends EventManager {
   constructor(connection) {
     super(connection);
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    this.runtimeProcess = globalThis?.process;
     this.producerConnected = false;
+    this.isRecoveringLeader = false;
+    this.seedBrokers = this.resolveBrokerList(this._connection.brokers ?? []);
+    this.groupId = this.runtimeProcess?.env?.KAFKA_GROUP || "default";
+    this.initializeKafkaClients(this.seedBrokers);
+    this.leaderSelectionPromise = this.pinToLeaderBroker();
+  }
+  async ensureProducerConnected() {
+    await this.waitForLeaderSelection();
+    if (this.producerConnected)
+      return;
+    await this.producer.connect();
+    this.producerConnected = true;
+  }
+  initializeKafkaClients(brokers) {
     this.kafka = new Kafka({
-      brokers: this._connection.brokers,
+      brokers,
       ssl: true,
       sasl: {
         mechanism: "scram-sha-256",
@@ -1713,14 +1733,195 @@ var KafkaManager = class extends EventManager {
       },
       logLevel: logLevel.ERROR
     });
-    this.consumer = this.kafka.consumer({ groupId: process.env.KAFKA_GROUP || "default" });
+    this.consumer = this.kafka.consumer({ groupId: this.groupId });
     this.producer = this.kafka.producer();
+    this.brokers = brokers;
+    this.producerConnected = false;
+    this.registerConsumerEventHandlers();
   }
-  async ensureProducerConnected() {
-    if (this.producerConnected)
+  async pinToLeaderBroker() {
+    try {
+      const leaderBroker = await this.fetchLeaderBroker(this.kafka);
+      if (!leaderBroker) {
+        console.warn("[KafkaManager] Unable to determine leader broker. Using configured list.", { brokers: this.brokers });
+        return;
+      }
+      if (this.brokers.length === 1 && this.brokers[0] === leaderBroker) {
+        return;
+      }
+      console.info("[KafkaManager] Leader broker resolved. Reinitializing clients", { leaderBroker });
+      this.initializeKafkaClients([leaderBroker]);
+    } catch (error) {
+      console.warn("[KafkaManager] Error resolving leader broker. Falling back to configured brokers.", error);
+    }
+  }
+  buildKafkaClient(brokers) {
+    return new Kafka({
+      brokers,
+      ssl: true,
+      sasl: {
+        mechanism: "scram-sha-256",
+        username: this._connection.userName,
+        password: this._connection.password
+      },
+      logLevel: logLevel.ERROR
+    });
+  }
+  async fetchLeaderBroker(kafkaClient) {
+    const admin = kafkaClient.admin();
+    try {
+      await admin.connect();
+      const cluster = await admin.describeCluster();
+      const controllerId = cluster?.controller;
+      const controller = controllerId !== void 0 ? cluster?.brokers?.find((node) => node.nodeId === controllerId) : void 0;
+      if (!controller?.host) {
+        return null;
+      }
+      const host = controller.host;
+      const port = controller.port ?? 9092;
+      return this.normalizeBroker(`${host}:${port}`);
+    } finally {
+      await admin.disconnect().catch(() => void 0);
+    }
+  }
+  async recoverFromLeaderChange(origin) {
+    if (this.isRecoveringLeader)
       return;
-    await this.producer.connect();
-    this.producerConnected = true;
+    this.isRecoveringLeader = true;
+    console.warn("[KafkaManager] Recovering from leader change", { origin, brokers: this.brokers });
+    await this.execCallback(this._onReconnect, {
+      message: "[KafkaManager] Leader change detected. Reconnecting to new leader.",
+      context: { origin }
+    });
+    try {
+      await this.stop().catch((error) => {
+        console.warn("[KafkaManager] Error stopping previous connection during recovery", error);
+      });
+      const seedClient = this.buildKafkaClient(this.seedBrokers);
+      const leaderBroker = await this.fetchLeaderBroker(seedClient).catch((error) => {
+        console.warn("[KafkaManager] Error resolving new leader broker during recovery", error);
+        return null;
+      });
+      const brokersToUse = leaderBroker ? [leaderBroker] : this.seedBrokers;
+      console.info("[KafkaManager] Reconnecting using brokers", { brokersToUse });
+      this.initializeKafkaClients(brokersToUse);
+      this.leaderSelectionPromise = Promise.resolve();
+      await this.run();
+    } catch (error) {
+      console.error("[KafkaManager] Failed to recover from leader change", { origin, error });
+      await this.execCallback(this._onError, error);
+    } finally {
+      this.isRecoveringLeader = false;
+    }
+  }
+  async waitForLeaderSelection() {
+    try {
+      await this.leaderSelectionPromise;
+    } catch {
+    }
+  }
+  resolveBrokerList(brokers) {
+    const envBrokers = (this.runtimeProcess?.env?.KAFKA_BROKERS ?? "").split(",").map((value) => value.trim()).filter(Boolean);
+    const normalized = [...brokers, ...envBrokers].map((value) => this.normalizeBroker(value)).filter((value) => Boolean(value));
+    const uniqueBrokers = Array.from(new Set(normalized));
+    if (!uniqueBrokers.length) {
+      throw new InternalError(ErrorManager.APP_ERRORS.PROCESS, "Kafka brokers configuration is empty.");
+    }
+    console.info("[KafkaManager] Using brokers", uniqueBrokers);
+    return uniqueBrokers;
+  }
+  normalizeBroker(broker) {
+    if (!broker)
+      return null;
+    const trimmed = broker.trim();
+    if (!trimmed)
+      return null;
+    if (trimmed.startsWith("PLAINTEXT://") || trimmed.startsWith("SSL://")) {
+      return trimmed;
+    }
+    return trimmed.includes(":") ? trimmed : `${trimmed}:9092`;
+  }
+  registerConsumerEventHandlers() {
+    const events = this.consumer.events;
+    this.consumer.on(events.REBALANCING, async (event) => {
+      await this.logConsumerEvent("REBALANCING", event);
+    });
+    const rebalancedEvent = this.consumer.events.REBALANCED;
+    if (rebalancedEvent) {
+      this.consumer.on(rebalancedEvent, async (event) => {
+        await this.logConsumerEvent("REBALANCED", event);
+      });
+    }
+    this.consumer.on(events.REQUEST_TIMEOUT, async (event) => {
+      await this.logConsumerEvent("REQUEST_TIMEOUT", event, "error");
+      this.handleKafkaError(event?.payload?.error, "consumer.REQUEST_TIMEOUT", { broker: event?.payload?.broker });
+    });
+    this.consumer.on(events.CRASH, async (event) => {
+      await this.logConsumerEvent("CRASH", event, "error");
+      this.handleKafkaError(event?.payload?.error, "consumer.CRASH", { broker: event?.payload?.broker });
+      if (event?.payload?.restart === false) {
+        void this.recoverFromLeaderChange("consumer.CRASH.no-restart");
+      }
+    });
+    this.consumer.on(events.DISCONNECT, async (event) => {
+      await this.logConsumerEvent("DISCONNECT", event, "error");
+    });
+    this.consumer.on(events.CONNECT, async (event) => {
+      await this.logConsumerEvent("CONNECT", event);
+    });
+  }
+  async logConsumerEvent(eventName, kafkaEvent, level = "info") {
+    const payload = kafkaEvent?.payload ?? {};
+    const broker = payload?.broker ?? payload?.coordinator ?? payload?.leaderId ?? null;
+    const error = payload?.error ? {
+      type: payload.error.type ?? payload.error.name,
+      message: payload.error.message
+    } : void 0;
+    const context = {
+      event: eventName,
+      timestamp: kafkaEvent?.timestamp ?? DateTime.now().value,
+      broker,
+      brokers: this.brokers,
+      groupId: payload?.groupId,
+      memberId: payload?.memberId,
+      leaderId: payload?.leaderId,
+      clientId: payload?.clientId,
+      topicPartitions: payload?.topicPartitions,
+      error
+    };
+    const message = `[KafkaManager][${eventName}] ${error?.message ?? ""}`.trim();
+    if (level === "error") {
+      console.error(message, context);
+      await this.execCallback(this._onError, { message, context });
+    } else {
+      console.info(message, context);
+      await this.execCallback(this._onReconnect, { message, context });
+    }
+  }
+  handleKafkaError(error, origin, metadata = {}) {
+    if (!error)
+      return;
+    const errorType = error.type ?? error.name;
+    const isNotLeaderError = errorType === "NOT_LEADER_FOR_PARTITION" || error?.name === "KafkaJSNotLeaderForPartition";
+    if (isNotLeaderError) {
+      const payload = {
+        origin,
+        broker: error?.broker ?? metadata?.broker,
+        topic: error?.topic,
+        partition: error?.partition,
+        metadata
+      };
+      console.error("[KafkaManager][NOT_LEADER_FOR_PARTITION] Reconnecting to new leader", payload);
+      void this.execCallback(this._onCrash, {
+        message: "[KafkaManager] NOT_LEADER_FOR_PARTITION detected. Reconnecting to new leader.",
+        context: payload
+      });
+      void this.recoverFromLeaderChange(origin);
+      return;
+    }
+    if (error instanceof KafkaJSRequestTimeoutError || errorType === "REQUEST_TIMEOUT") {
+      console.warn("[KafkaManager][REQUEST_TIMEOUT]", { origin, metadata });
+    }
   }
   async commitMessage(topic, partition, message) {
     const next = (BigInt(message.offset) + 1n).toString();
@@ -1728,9 +1929,10 @@ var KafkaManager = class extends EventManager {
   }
   async run(autocommit = false) {
     const __run = async () => {
+      await this.waitForLeaderSelection();
       await this.execCallback(this._onStart, { message: `--- ${this.constructor.name} started ---` });
       await this.consumer.connect();
-      await this.execCallback(this._onConnected, { message: `--- ${this.constructor.name} connected to ${this._connection.brokers} ---` });
+      await this.execCallback(this._onConnected, { message: `--- ${this.constructor.name} connected to ${this.brokers.join(", ")} ---` });
       for (let topic of this._topics) {
         try {
           await this.consumer.subscribe({ topic, fromBeginning: false });
@@ -1767,6 +1969,7 @@ var KafkaManager = class extends EventManager {
             await heartbeat();
           } catch (error) {
             await this.execCallback(this._onError, error);
+            this.handleKafkaError(error, "consumer.eachMessage", { topic, partition });
             throw error;
           }
         }
@@ -1779,6 +1982,7 @@ var KafkaManager = class extends EventManager {
     }
   }
   async send(e) {
+    await this.waitForLeaderSelection();
     const evt = {
       date: DateTime.now().value,
       tenant: e.tenant.value,
@@ -1807,9 +2011,6 @@ var KafkaManager = class extends EventManager {
     }
   }
   async start(autocommit = false) {
-    this.consumer.on(this.consumer.events.CRASH, async (error) => {
-      await this.execCallback(this._onError, new InternalError(ErrorManager.APP_ERRORS.PROCESS, error.payload.error.stack));
-    });
     await this.run();
   }
   async pause() {