pulse-sdk 0.0.3 → 0.0.4

package/README.md

@@ -149,6 +149,25 @@ The tests in `tests/` start a lightweight gRPC test server automatically; run th
 - Can I use decorators for handlers? Yes — TypeScript supports decorators, but this SDK does not use them by default. We can add decorator helpers later if desired.
 - How do I enable TLS / secure credentials? The client factory currently uses `createInsecure()` by default. We can expose a `credentials` option on `PulseConfig` to accept `grpc.credentials.createSsl(...)` or other credential objects.
 
+**Grouped consumption**
+
+- **What it does:** When `grouped` is `true` (the default) the SDK coalesces consumers inside the same process that use the same `consumerName` into a single streaming connection. Messages are distributed (round-robin) among the registered handlers so each message is delivered to only one handler in the group (1x per client id).
+- **Why:** This mirrors the Python SDK behavior where handlers registered with `grouped=True` share a single consumer stream and avoid duplicate processing inside the same client.
+- **How to configure:**
+	- `pulse.yml` (recommended): set `grouped: true` or `grouped: false` under top-level config.
+	- Environment variable: `PULSE_GROUPED=true|false`.
+	- Programmatically: pass `grouped` in the `PulseConfig` passed to `Producer`/`Consumer`.
+- **Default:** `grouped` defaults to `true`.
+- **Grouped=false behavior:** When `grouped` is `false`, the SDK will ensure consumers use unique consumer IDs (if you pass the default client name), so multiple consumers in the same process each receive all messages independently (useful for testing or when you want duplicate consumption).
+- **Example `pulse.yml` entry:**
+
+```yaml
+grpcUrl: localhost:5556
+grouped: true
+```
+
+The test-suite includes integration tests that validate both `grouped=true` and `grouped=false` behaviour.
+
 ## Contributing
 
 Please open a PR with tests. The existing tests validate basic Producer/Consumer behaviour.

package/dist/config.d.ts

@@ -8,6 +8,7 @@ export interface PulseConfig {
     grpcUrl: string;
     eventTypes: string[];
     consumerName?: string;
+    grouped?: boolean;
     topics?: TopicConfig[];
 }
 export declare function loadConfig(configPath?: string): PulseConfig;

package/dist/config.js

@@ -13,6 +13,7 @@ const client_1 = require("./proto/client");
 const DEFAULTS = {
     grpcUrl: 'localhost:50052',
     eventTypes: ['events'],
+    grouped: true,
 };
 function loadConfig(configPath) {
     const candidates = [
@@ -37,10 +38,14 @@ function loadConfig(configPath) {
         envCfg.eventTypes = process.env.PULSE_EVENT_TYPES.split(',');
     if (process.env.PULSE_CONSUMER_NAME)
         envCfg.consumerName = process.env.PULSE_CONSUMER_NAME;
+    if (process.env.PULSE_GROUPED)
+        envCfg.grouped = process.env.PULSE_GROUPED === 'true';
     const merged = Object.assign({}, DEFAULTS, fileCfg, envCfg);
     // Ensure eventTypes array exists
     if (!merged.eventTypes)
         merged.eventTypes = DEFAULTS.eventTypes;
+    if (merged.grouped === undefined)
+        merged.grouped = true;
     return merged;
 }
 // Initialize topics from config using gRPC CreateTopic RPC.

package/dist/consumer.d.ts

@@ -4,7 +4,9 @@ export declare class Consumer {
     private config;
     private handlers;
     private client;
+    private unregisterFns;
     constructor(config: PulseConfig);
     on(eventType: string, handler: EventHandler): void;
     start(topic?: string, consumerName?: string): Promise<void>;
+    close(): void;
 }

package/dist/consumer.js

@@ -2,10 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Consumer = void 0;
 const client_1 = require("./proto/client");
+const consumerManager_1 = require("./consumerManager");
+const crypto_1 = require("crypto");
+const message_1 = require("./message");
 class Consumer {
     constructor(config) {
         this.config = config;
         this.handlers = {};
+        this.unregisterFns = [];
         this.client = (0, client_1.createClient)(config.grpcUrl);
     }
     on(eventType, handler) {
@@ -15,24 +19,81 @@ class Consumer {
         this.handlers[eventType].push(handler);
     }
     async start(topic, consumerName = 'default') {
-        const req = { topic: topic || this.config.eventTypes[0], consumer_name: consumerName, offset: 0 };
-        const stream = this.client.Consume(req);
-        stream.on('data', (msg) => {
-            const payloadBuf = msg.payload;
-            let parsed = payloadBuf;
-            try {
-                parsed = JSON.parse(payloadBuf.toString());
+        const topicName = topic || this.config.eventTypes[0];
+        // If grouped is enabled (default true) use shared in-process stream,
+        // otherwise create a dedicated stream (each consumer gets all messages).
+        const grouped = this.config.grouped !== false;
+        if (grouped) {
+            // register handlers for this topic to shared stream
+            const handlers = this.handlers[topicName] || [];
+            for (const h of handlers) {
+                const unregister = (0, consumerManager_1.registerSharedHandler)(this.config.grpcUrl, topicName, consumerName, (msg, stub, offset) => {
+                    // run handler with context so commit() works
+                    // debug: console.log('consumer.wrapper.invoke', consumerName, topicName);
+                    (0, message_1.runWithContext)({ stub: stub || this.client, topic: topicName, consumerName, offset: offset ?? msg.offset }, () => {
+                        try {
+                            h(msg);
+                        }
+                        catch (e) { /* ignore */ }
+                    });
+                });
+                this.unregisterFns.push(unregister);
             }
-            catch (e) {
-                // keep raw buffer if not JSON
+            // Return a promise that never resolves (stream runs until process exits)
+            return new Promise(() => { });
+        }
+        // If grouped is explicitly false, and the consumerName equals the configured
+        // client name or the default literal, generate a unique consumer id so each
+        // consumer receives messages independently (mirrors Python behaviour).
+        if (!grouped) {
+            const base = this.config.consumerName || 'default';
+            if (consumerName === base || consumerName === 'default') {
+                consumerName = `${base}-${(0, crypto_1.randomUUID)().replace(/-/g, '')}`;
             }
+        }
+        const req = { topic: topicName, consumer_name: consumerName, offset: 0 };
+        let stream = null;
+        try {
+            stream = this.client.Consume(req);
+        }
+        catch (err) {
+            // If the client throws synchronously (e.g. CANCELLED), return a promise
+            // that is rejected but handled to avoid an unhandled rejection when
+            // callers don't await `start()` (tests call start() without awaiting).
+            const p = Promise.reject(err);
+            p.catch(() => { }); // swallow to avoid uncaught rejection
+            return p;
+        }
+        stream.on('data', (msg) => {
+            const message = new message_1.Message(msg);
             const handlers = this.handlers[req.topic] || [];
-            handlers.forEach(h => h({ offset: msg.offset, timestamp: msg.timestamp, payload: parsed, headers: msg.headers }));
+            for (const h of handlers) {
+                // run handler within AsyncLocalStorage context so commit() can access stub and offset
+                (0, message_1.runWithContext)({ stub: this.client, topic: req.topic, consumerName, offset: message.offset }, () => {
+                    try {
+                        h(message);
+                    }
+                    catch (e) { /* handler error ignored here */ }
+                });
+            }
         });
-        return new Promise((resolve, reject) => {
+        const p = new Promise((resolve, reject) => {
             stream.on('end', () => resolve());
             stream.on('error', (e) => reject(e));
         });
+        // prevent unhandled rejections when callers don't await the returned promise
+        p.catch(() => { });
+        return p;
+    }
+    // unregister any shared handlers when this consumer is discarded
+    close() {
+        for (const u of this.unregisterFns) {
+            try {
+                u();
+            }
+            catch (e) { /* ignore */ }
+        }
+        this.unregisterFns = [];
     }
 }
 exports.Consumer = Consumer;

package/dist/consumerManager.d.ts

@@ -0,0 +1,4 @@
+type Handler = (msg: any) => void;
+export declare function shutdownAll(): Promise<void>;
+export declare function registerSharedHandler(grpcUrl: string, topic: string, consumerName: string, handler: Handler): () => void;
+export {};

package/dist/consumerManager.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shutdownAll = shutdownAll;
+exports.registerSharedHandler = registerSharedHandler;
+const client_1 = require("./proto/client");
+const message_1 = require("./message");
+let suppressStreamWarnings = false;
+async function shutdownAll() {
+    // When shutting down tests/teardown, suppress stream warnings so Jest doesn't
+    // complain about logs after tests are finished.
+    suppressStreamWarnings = true;
+    for (const [k, entry] of Array.from(registry.entries())) {
+        try {
+            if (entry.stream) {
+                try {
+                    entry.stream.removeAllListeners();
+                }
+                catch (_) { }
+                try {
+                    if (entry.stream.cancel)
+                        entry.stream.cancel();
+                }
+                catch (_) { }
+                entry.stream = null;
+            }
+            try {
+                entry.handlers.clear();
+            }
+            catch (_) { }
+        }
+        catch (_) {
+            // ignore individual errors
+        }
+        try {
+            if (entry.client && typeof entry.client.close === 'function')
+                entry.client.close();
+        }
+        catch (_) { }
+        try {
+            registry.delete(k);
+        }
+        catch (_) { }
+    }
+}
+const registry = new Map();
+function keyFor(grpcUrl, topic, consumerName) {
+    return `${grpcUrl}::${topic}::${consumerName}`;
+}
+function registerSharedHandler(grpcUrl, topic, consumerName, handler) {
+    const k = keyFor(grpcUrl, topic, consumerName);
+    let entry = registry.get(k);
+    if (!entry) {
+        const client = (0, client_1.createClient)(grpcUrl);
+        entry = { topic, consumerName, client, stream: null, handlers: new Set(), nextIndex: 0, grpcUrl };
+        // add handler before starting the stream to avoid losing early messages
+        entry.handlers.add(handler);
+        registry.set(k, entry);
+        startStream(entry);
+        return () => {
+            // unregister
+            const e = registry.get(k);
+            if (!e)
+                return;
+            e.handlers.delete(handler);
+            if (e.handlers.size === 0) {
+                try {
+                    if (e.stream && e.stream.cancel)
+                        e.stream.cancel();
+                }
+                catch (err) { }
+                registry.delete(k);
+            }
+        };
+    }
+    entry.handlers.add(handler);
+    return () => {
+        // unregister
+        const e = registry.get(k);
+        if (!e)
+            return;
+        e.handlers.delete(handler);
+        if (e.handlers.size === 0) {
+            // cleanup stream
+            try {
+                if (e.stream && e.stream.cancel)
+                    e.stream.cancel();
+            }
+            catch (e) {
+                // ignore
+            }
+            registry.delete(k);
+        }
+    };
+}
+function startStream(entry) {
+    const req = { topic: entry.topic, consumer_name: entry.consumerName, offset: 0 };
+    const stream = entry.client.Consume(req);
+    entry.stream = stream;
+    // debug: console.log('consumerManager.startStream', entry.grpcUrl, entry.topic, entry.consumerName);
+    stream.on('data', (msg) => {
+        const message = new message_1.Message(msg);
+        const handlers = Array.from(entry.handlers);
+        if (handlers.length === 0)
+            return;
+        if (entry.nextIndex === undefined)
+            entry.nextIndex = 0;
+        const h = handlers[entry.nextIndex % handlers.length];
+        entry.nextIndex = (entry.nextIndex + 1) % handlers.length;
+        try {
+            // run handler inside context providing the stub for commit()
+            // debug: console.log('consumerManager.dispatch', entry.topic, 'offset', message.offset, 'handlerIndex', entry.nextIndex);
+            (0, message_1.runWithContext)({ stub: entry.client, topic: entry.topic, consumerName: entry.consumerName, offset: message.offset }, () => {
+                h(message);
+            });
+        }
+        catch (e) {
+            console.warn('handler error', e);
+        }
+    });
+    stream.on('error', (e) => {
+        // Log once and clean up the registry entry to avoid reconnect storms and test leaks
+        if (!suppressStreamWarnings) {
+            // Ignore normal client-side cancellations which happen during unregister
+            // and shutdown; only log unexpected stream errors.
+            try {
+                const code = e && typeof e.code !== 'undefined' ? e.code : null;
+                if (code !== 1) {
+                    console.warn('shared consumer stream error for', entry.topic, e);
+                }
+            }
+            catch (_) {
+                // if anything goes wrong determining code, log the error
+                console.warn('shared consumer stream error for', entry.topic, e);
+            }
+        }
+        try {
+            if (entry.stream) {
+                try {
+                    entry.stream.removeAllListeners();
+                }
+                catch (_) { }
+                try {
+                    if (entry.stream.cancel)
+                        entry.stream.cancel();
+                }
+                catch (_) { }
+                entry.stream = null;
+            }
+        }
+        catch (err) {
+            // ignore
+        }
+        try {
+            try {
+                if (entry.client && typeof entry.client.close === 'function')
+                    entry.client.close();
+            }
+            catch (_) { }
+            registry.delete(keyFor(entry.grpcUrl || '', entry.topic, entry.consumerName));
+        }
+        catch (err) {
+            // ignore
+        }
+    });
+    stream.on('end', () => {
+        // stream ended; clean up entry
+        try {
+            if (entry.stream) {
+                try {
+                    entry.stream.removeAllListeners();
+                }
+                catch (_) { }
+                try {
+                    if (entry.stream.cancel)
+                        entry.stream.cancel();
+                }
+                catch (_) { }
+                entry.stream = null;
+            }
+            try {
+                if (entry.client && typeof entry.client.close === 'function')
+                    entry.client.close();
+            }
+            catch (_) { }
+            registry.delete(keyFor(entry.grpcUrl || '', entry.topic, entry.consumerName));
+        }
+        catch (err) {
+            // ignore
+        }
+    });
+}

package/dist/index.d.ts

@@ -1,3 +1,5 @@
 export * from './producer';
 export * from './consumer';
 export * from './config';
+export * from './message';
+export * from './consumerManager';

package/dist/index.js

@@ -17,3 +17,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./producer"), exports);
 __exportStar(require("./consumer"), exports);
 __exportStar(require("./config"), exports);
+__exportStar(require("./message"), exports);
+__exportStar(require("./consumerManager"), exports);

package/dist/message.d.ts

@@ -0,0 +1,24 @@
+export interface IMessage {
+    offset: number;
+    timestamp: number;
+    payload: any;
+    headers: Record<string, string>;
+}
+interface MessageContext {
+    stub: any;
+    topic: string;
+    consumerName: string;
+    offset: number;
+    committed?: boolean;
+}
+export declare function runWithContext(ctx: MessageContext, fn: () => void): void;
+export declare function getContext(): MessageContext | undefined;
+export declare function commit(): Promise<void>;
+export declare class Message implements IMessage {
+    offset: number;
+    timestamp: number;
+    payload: any;
+    headers: Record<string, string>;
+    constructor(protoMsg: any);
+}
+export {};

package/dist/message.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Message = void 0;
+exports.runWithContext = runWithContext;
+exports.getContext = getContext;
+exports.commit = commit;
+const async_hooks_1 = require("async_hooks");
+const storage = new async_hooks_1.AsyncLocalStorage();
+function runWithContext(ctx, fn) {
+    storage.run(ctx, fn);
+}
+function getContext() {
+    return storage.getStore();
+}
+async function commit() {
+    const ctx = storage.getStore();
+    if (!ctx)
+        throw new Error('commit() called outside of a consumer handler');
+    if (ctx.committed)
+        return;
+    return new Promise((resolve, reject) => {
+        try {
+            ctx.stub.CommitOffset({ topic: ctx.topic, consumer_name: ctx.consumerName, offset: ctx.offset + 1 }, (err, res) => {
+                if (err)
+                    return reject(err);
+                ctx.committed = true;
+                resolve();
+            });
+        }
+        catch (e) {
+            reject(e);
+        }
+    });
+}
+class Message {
+    constructor(protoMsg) {
+        this.offset = protoMsg.offset;
+        this.timestamp = protoMsg.timestamp;
+        this.headers = {};
+        try {
+            this.headers = Object.assign({}, protoMsg.headers);
+        }
+        catch (e) {
+            this.headers = {};
+        }
+        const buf = protoMsg.payload;
+        const ptype = this.headers['payload-type'];
+        if (ptype === 'json') {
+            try {
+                this.payload = JSON.parse(buf.toString());
+            }
+            catch (e) {
+                this.payload = buf;
+            }
+        }
+        else if (ptype === 'string') {
+            try {
+                this.payload = buf.toString('utf8');
+            }
+            catch (e) {
+                this.payload = buf;
+            }
+        }
+        else if (ptype === 'bytes') {
+            this.payload = buf;
+        }
+        else {
+            try {
+                this.payload = JSON.parse(buf.toString());
+            }
+            catch (e) {
+                this.payload = buf;
+            }
+        }
+    }
+}
+exports.Message = Message;

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "pulse-sdk",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "Pulse SDK for Node.js/TypeScript",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "build": "tsc",
     "postbuild": "node scripts/copy-proto.js",
-    "test": "jest"
+    "test": "npm run build && jest"
   },
   "author": "",
   "license": "MIT",
@@ -22,21 +22,21 @@
   "devDependencies": {
     "typescript": "^5.0.0",
     "jest": "^29.0.0",
-    "ts-jest": "^29.0.0",
-    "@types/jest": "^29.0.0",
+    "ts-jest": "^29.0.0"
+    ,"@types/jest": "^29.0.0",
     "@types/js-yaml": "^4.0.5"
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.8.0",
     "protobufjs": "^7.2.0",
-    "@grpc/proto-loader": "^0.7.0",
-    "js-yaml": "^4.1.0"
-  },
+    "@grpc/proto-loader": "^0.7.0"
+    ,"js-yaml": "^4.1.0"
+  }
+,
   "jest": {
     "preset": "ts-jest",
     "testEnvironment": "node",
-    "testMatch": [
-      "**/tests/**/*.test.ts"
-    ]
+    "testMatch": ["**/tests/**/*.test.ts"],
+    "globalTeardown": "<rootDir>/tests/globalTeardown.js"
   }
 }