pulse-sdk 0.0.2 → 0.0.4

package/README.md

@@ -28,11 +28,58 @@ Run the lightweight integration tests (a test gRPC server is started automatical
 npm test
 ```
 
-## Configuration (current)
+## Configuration
 
-Currently the SDK supports programmatic configuration only: create and pass a `PulseConfig` object when instantiating `Producer` or `Consumer`.
+The SDK supports configuration via (in priority order):
 
-The code exposes a `PulseConfig` interface for strong typing. Environment-file loading (e.g. `pulse.yml` or `PULSE_*` environment variables) is planned but not implemented in this initial version — for now, pass configuration directly in code.
+1. Programmatic: pass a `PulseConfig` object when creating `Producer` or `Consumer`.
+2. Environment variables: `PULSE_GRPC_URL`, `PULSE_EVENT_TYPES` (comma-separated), `PULSE_CONSUMER_NAME`.
+3. File: a `pulse.yml` or `pulse.yaml` file placed at the process working directory or in `~/.pulse/pulse.yml`.
+
+The SDK exposes two helpers:
+
+- `loadConfig(configPath?: string): PulseConfig` — reads `pulse.yml` / env and returns a merged config object.
+- `initFromConfig(cfg: PulseConfig): Promise<void>` — calls the broker `CreateTopic` RPC for every topic declared in `cfg.topics`. This lets the SDK create any required topics automatically at startup (useful for tests or first-run).
+
+Example `pulse.yml`:
+
+```yaml
+grpcUrl: localhost:5556
+eventTypes:
+	- events
+	- transactions
+consumerName: my-consumer
+topics:
+	- name: events
+		fifo: false
+	- name: transactions
+		fifo: true
+```
+
+Usage example (auto-create topics then start):
+
+```ts
+import { loadConfig, initFromConfig, Producer, Consumer } from 'pulse-sdk';
+
+async function main() {
+	const cfg = loadConfig(); // reads pulse.yml or environment
+	await initFromConfig(cfg); // create topics listed in cfg.topics (no-op if none)
+
+	const producer = new Producer(cfg);
+	await producer.send('events', { type: 'user.created', id: 1 });
+
+	const consumer = new Consumer(cfg);
+	consumer.on('events', (msg) => console.log('received', msg.payload));
+	await consumer.start('events', cfg.consumerName || 'default');
+}
+
+main().catch(console.error);
+```
+
+Notes:
+- `initFromConfig` calls the gRPC `CreateTopic` RPC. If the broker responds with an error for a topic that already exists, the SDK logs a warning and continues.
+- Ensure the broker is running and reachable at `cfg.grpcUrl` before calling `initFromConfig`.
+- The SDK bundles `pulse.proto` inside the package so consumers do not need to copy proto files into their projects.
 
 ## Quick Start
 
@@ -102,6 +149,25 @@ The tests in `tests/` start a lightweight gRPC test server automatically; run th
 - Can I use decorators for handlers? Yes — TypeScript supports decorators, but this SDK does not use them by default. We can add decorator helpers later if desired.
 - How do I enable TLS / secure credentials? The client factory currently uses `createInsecure()` by default. We can expose a `credentials` option on `PulseConfig` to accept `grpc.credentials.createSsl(...)` or other credential objects.
 
+**Grouped consumption**
+
+- **What it does:** When `grouped` is `true` (the default) the SDK coalesces consumers inside the same process that use the same `consumerName` into a single streaming connection. Messages are distributed (round-robin) among the registered handlers so each message is delivered to only one handler in the group (1x per client id).
+- **Why:** This mirrors the Python SDK behavior where handlers registered with `grouped=True` share a single consumer stream and avoid duplicate processing inside the same client.
+- **How to configure:**
+	- `pulse.yml` (recommended): set `grouped: true` or `grouped: false` under top-level config.
+	- Environment variable: `PULSE_GROUPED=true|false`.
+	- Programmatically: pass `grouped` in the `PulseConfig` passed to `Producer`/`Consumer`.
+- **Default:** `grouped` defaults to `true`.
+- **Grouped=false behavior:** When `grouped` is `false`, the SDK will ensure consumers use unique consumer IDs (if you pass the default client name), so multiple consumers in the same process each receive all messages independently (useful for testing or when you want duplicate consumption).
+- **Example `pulse.yml` entry:**
+
+```yaml
+grpcUrl: localhost:5556
+grouped: true
+```
+
+The test-suite includes integration tests that validate both `grouped=true` and `grouped=false` behaviour.
+
 ## Contributing
 
 Please open a PR with tests. The existing tests validate basic Producer/Consumer behaviour.

package/dist/config.d.ts

@@ -1,5 +1,15 @@
+export interface TopicConfig {
+    name: string;
+    fifo?: boolean;
+    retention_bytes?: number;
+    retention_time?: number;
+}
 export interface PulseConfig {
     grpcUrl: string;
     eventTypes: string[];
+    consumerName?: string;
+    grouped?: boolean;
+    topics?: TopicConfig[];
 }
-export declare function loadConfig(config: PulseConfig): PulseConfig;
+export declare function loadConfig(configPath?: string): PulseConfig;
+export declare function initFromConfig(cfg: PulseConfig): Promise<void>;

package/dist/config.js

@@ -1,7 +1,76 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadConfig = loadConfig;
-function loadConfig(config) {
-    // Aqui pode ser expandido para ler de arquivo ou env
-    return config;
+exports.initFromConfig = initFromConfig;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const client_1 = require("./proto/client");
+const DEFAULTS = {
+    grpcUrl: 'localhost:50052',
+    eventTypes: ['events'],
+    grouped: true,
+};
+function loadConfig(configPath) {
+    const candidates = [
+        configPath,
+        path_1.default.resolve(process.cwd(), 'pulse.yml'),
+        path_1.default.resolve(process.cwd(), 'pulse.yaml'),
+        path_1.default.join(os_1.default.homedir(), '.pulse', 'pulse.yml'),
+    ].filter(Boolean);
+    let fileCfg = {};
+    for (const p of candidates) {
+        if (p && fs_1.default.existsSync(p)) {
+            const raw = fs_1.default.readFileSync(p, 'utf8');
+            const parsed = js_yaml_1.default.load(raw);
+            fileCfg = parsed || {};
+            break;
+        }
+    }
+    const envCfg = {};
+    if (process.env.PULSE_GRPC_URL)
+        envCfg.grpcUrl = process.env.PULSE_GRPC_URL;
+    if (process.env.PULSE_EVENT_TYPES)
+        envCfg.eventTypes = process.env.PULSE_EVENT_TYPES.split(',');
+    if (process.env.PULSE_CONSUMER_NAME)
+        envCfg.consumerName = process.env.PULSE_CONSUMER_NAME;
+    if (process.env.PULSE_GROUPED)
+        envCfg.grouped = process.env.PULSE_GROUPED === 'true';
+    const merged = Object.assign({}, DEFAULTS, fileCfg, envCfg);
+    // Ensure eventTypes array exists
+    if (!merged.eventTypes)
+        merged.eventTypes = DEFAULTS.eventTypes;
+    if (merged.grouped === undefined)
+        merged.grouped = true;
+    return merged;
+}
+// Initialize topics from config using gRPC CreateTopic RPC.
+async function initFromConfig(cfg) {
+    if (!cfg.topics || cfg.topics.length === 0)
+        return;
+    const client = (0, client_1.createClient)(cfg.grpcUrl);
+    for (const t of cfg.topics) {
+        const req = {
+            topic: t.name || t['name'],
+            fifo: !!t.fifo,
+            retention_bytes: t.retention_bytes || 0,
+            retention_time: t.retention_time || 0,
+        };
+        await new Promise((resolve, reject) => {
+            client.CreateTopic(req, (err, res) => {
+                if (err) {
+                    // If topic exists the server may return an error; log and continue
+                    console.warn('CreateTopic error for', req.topic, err.message || err);
+                    resolve();
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
 }

package/dist/consumer.d.ts

@@ -4,7 +4,9 @@ export declare class Consumer {
     private config;
     private handlers;
     private client;
+    private unregisterFns;
     constructor(config: PulseConfig);
     on(eventType: string, handler: EventHandler): void;
     start(topic?: string, consumerName?: string): Promise<void>;
+    close(): void;
 }

package/dist/consumer.js

@@ -2,10 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Consumer = void 0;
 const client_1 = require("./proto/client");
+const consumerManager_1 = require("./consumerManager");
+const crypto_1 = require("crypto");
+const message_1 = require("./message");
 class Consumer {
     constructor(config) {
         this.config = config;
         this.handlers = {};
+        this.unregisterFns = [];
         this.client = (0, client_1.createClient)(config.grpcUrl);
     }
     on(eventType, handler) {
@@ -15,24 +19,81 @@ class Consumer {
         this.handlers[eventType].push(handler);
     }
     async start(topic, consumerName = 'default') {
-        const req = { topic: topic || this.config.eventTypes[0], consumer_name: consumerName, offset: 0 };
-        const stream = this.client.Consume(req);
-        stream.on('data', (msg) => {
-            const payloadBuf = msg.payload;
-            let parsed = payloadBuf;
-            try {
-                parsed = JSON.parse(payloadBuf.toString());
+        const topicName = topic || this.config.eventTypes[0];
+        // If grouped is enabled (default true) use shared in-process stream,
+        // otherwise create a dedicated stream (each consumer gets all messages).
+        const grouped = this.config.grouped !== false;
+        if (grouped) {
+            // register handlers for this topic to shared stream
+            const handlers = this.handlers[topicName] || [];
+            for (const h of handlers) {
+                const unregister = (0, consumerManager_1.registerSharedHandler)(this.config.grpcUrl, topicName, consumerName, (msg, stub, offset) => {
+                    // run handler with context so commit() works
+                    // debug: console.log('consumer.wrapper.invoke', consumerName, topicName);
+                    (0, message_1.runWithContext)({ stub: stub || this.client, topic: topicName, consumerName, offset: offset ?? msg.offset }, () => {
+                        try {
+                            h(msg);
+                        }
+                        catch (e) { /* ignore */ }
+                    });
+                });
+                this.unregisterFns.push(unregister);
             }
-            catch (e) {
-                // keep raw buffer if not JSON
+            // Return a promise that never resolves (stream runs until process exits)
+            return new Promise(() => { });
+        }
+        // If grouped is explicitly false, and the consumerName equals the configured
+        // client name or the default literal, generate a unique consumer id so each
+        // consumer receives messages independently (mirrors Python behaviour).
+        if (!grouped) {
+            const base = this.config.consumerName || 'default';
+            if (consumerName === base || consumerName === 'default') {
+                consumerName = `${base}-${(0, crypto_1.randomUUID)().replace(/-/g, '')}`;
             }
+        }
+        const req = { topic: topicName, consumer_name: consumerName, offset: 0 };
+        let stream = null;
+        try {
+            stream = this.client.Consume(req);
+        }
+        catch (err) {
+            // If the client throws synchronously (e.g. CANCELLED), return a promise
+            // that is rejected but handled to avoid an unhandled rejection when
+            // callers don't await `start()` (tests call start() without awaiting).
+            const p = Promise.reject(err);
+            p.catch(() => { }); // swallow to avoid uncaught rejection
+            return p;
+        }
+        stream.on('data', (msg) => {
+            const message = new message_1.Message(msg);
             const handlers = this.handlers[req.topic] || [];
-            handlers.forEach(h => h({ offset: msg.offset, timestamp: msg.timestamp, payload: parsed, headers: msg.headers }));
+            for (const h of handlers) {
+                // run handler within AsyncLocalStorage context so commit() can access stub and offset
+                (0, message_1.runWithContext)({ stub: this.client, topic: req.topic, consumerName, offset: message.offset }, () => {
+                    try {
+                        h(message);
+                    }
+                    catch (e) { /* handler error ignored here */ }
+                });
+            }
         });
-        return new Promise((resolve, reject) => {
+        const p = new Promise((resolve, reject) => {
             stream.on('end', () => resolve());
             stream.on('error', (e) => reject(e));
         });
+        // prevent unhandled rejections when callers don't await the returned promise
+        p.catch(() => { });
+        return p;
+    }
+    // unregister any shared handlers when this consumer is discarded
+    close() {
+        for (const u of this.unregisterFns) {
+            try {
+                u();
+            }
+            catch (e) { /* ignore */ }
+        }
+        this.unregisterFns = [];
     }
 }
 exports.Consumer = Consumer;

package/dist/consumerManager.d.ts

@@ -0,0 +1,4 @@
+type Handler = (msg: any) => void;
+export declare function shutdownAll(): Promise<void>;
+export declare function registerSharedHandler(grpcUrl: string, topic: string, consumerName: string, handler: Handler): () => void;
+export {};

package/dist/consumerManager.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shutdownAll = shutdownAll;
+exports.registerSharedHandler = registerSharedHandler;
+const client_1 = require("./proto/client");
+const message_1 = require("./message");
+let suppressStreamWarnings = false;
+async function shutdownAll() {
+    // When shutting down tests/teardown, suppress stream warnings so Jest doesn't
+    // complain about logs after tests are finished.
+    suppressStreamWarnings = true;
+    for (const [k, entry] of Array.from(registry.entries())) {
+        try {
+            if (entry.stream) {
+                try {
+                    entry.stream.removeAllListeners();
+                }
+                catch (_) { }
+                try {
+                    if (entry.stream.cancel)
+                        entry.stream.cancel();
+                }
+                catch (_) { }
+                entry.stream = null;
+            }
+            try {
+                entry.handlers.clear();
+            }
+            catch (_) { }
+        }
+        catch (_) {
+            // ignore individual errors
+        }
+        try {
+            if (entry.client && typeof entry.client.close === 'function')
+                entry.client.close();
+        }
+        catch (_) { }
+        try {
+            registry.delete(k);
+        }
+        catch (_) { }
+    }
+}
+const registry = new Map();
+function keyFor(grpcUrl, topic, consumerName) {
+    return `${grpcUrl}::${topic}::${consumerName}`;
+}
+function registerSharedHandler(grpcUrl, topic, consumerName, handler) {
+    const k = keyFor(grpcUrl, topic, consumerName);
+    let entry = registry.get(k);
+    if (!entry) {
+        const client = (0, client_1.createClient)(grpcUrl);
+        entry = { topic, consumerName, client, stream: null, handlers: new Set(), nextIndex: 0, grpcUrl };
+        // add handler before starting the stream to avoid losing early messages
+        entry.handlers.add(handler);
+        registry.set(k, entry);
+        startStream(entry);
+        return () => {
+            // unregister
+            const e = registry.get(k);
+            if (!e)
+                return;
+            e.handlers.delete(handler);
+            if (e.handlers.size === 0) {
+                try {
+                    if (e.stream && e.stream.cancel)
+                        e.stream.cancel();
+                }
+                catch (err) { }
+                registry.delete(k);
+            }
+        };
+    }
+    entry.handlers.add(handler);
+    return () => {
+        // unregister
+        const e = registry.get(k);
+        if (!e)
+            return;
+        e.handlers.delete(handler);
+        if (e.handlers.size === 0) {
+            // cleanup stream
+            try {
+                if (e.stream && e.stream.cancel)
+                    e.stream.cancel();
+            }
+            catch (e) {
+                // ignore
+            }
+            registry.delete(k);
+        }
+    };
+}
+function startStream(entry) {
+    const req = { topic: entry.topic, consumer_name: entry.consumerName, offset: 0 };
+    const stream = entry.client.Consume(req);
+    entry.stream = stream;
+    // debug: console.log('consumerManager.startStream', entry.grpcUrl, entry.topic, entry.consumerName);
+    stream.on('data', (msg) => {
+        const message = new message_1.Message(msg);
+        const handlers = Array.from(entry.handlers);
+        if (handlers.length === 0)
+            return;
+        if (entry.nextIndex === undefined)
+            entry.nextIndex = 0;
+        const h = handlers[entry.nextIndex % handlers.length];
+        entry.nextIndex = (entry.nextIndex + 1) % handlers.length;
+        try {
+            // run handler inside context providing the stub for commit()
+            // debug: console.log('consumerManager.dispatch', entry.topic, 'offset', message.offset, 'handlerIndex', entry.nextIndex);
+            (0, message_1.runWithContext)({ stub: entry.client, topic: entry.topic, consumerName: entry.consumerName, offset: message.offset }, () => {
+                h(message);
+            });
+        }
+        catch (e) {
+            console.warn('handler error', e);
+        }
+    });
+    stream.on('error', (e) => {
+        // Log once and clean up the registry entry to avoid reconnect storms and test leaks
+        if (!suppressStreamWarnings) {
+            // Ignore normal client-side cancellations which happen during unregister
+            // and shutdown; only log unexpected stream errors.
+            try {
+                const code = e && typeof e.code !== 'undefined' ? e.code : null;
+                if (code !== 1) {
+                    console.warn('shared consumer stream error for', entry.topic, e);
+                }
+            }
+            catch (_) {
+                // if anything goes wrong determining code, log the error
+                console.warn('shared consumer stream error for', entry.topic, e);
+            }
+        }
+        try {
+            if (entry.stream) {
+                try {
+                    entry.stream.removeAllListeners();
+                }
+                catch (_) { }
+                try {
+                    if (entry.stream.cancel)
+                        entry.stream.cancel();
+                }
+                catch (_) { }
+                entry.stream = null;
+            }
+        }
+        catch (err) {
+            // ignore
+        }
+        try {
+            try {
+                if (entry.client && typeof entry.client.close === 'function')
+                    entry.client.close();
+            }
+            catch (_) { }
+            registry.delete(keyFor(entry.grpcUrl || '', entry.topic, entry.consumerName));
+        }
+        catch (err) {
+            // ignore
+        }
+    });
+    stream.on('end', () => {
+        // stream ended; clean up entry
+        try {
+            if (entry.stream) {
+                try {
+                    entry.stream.removeAllListeners();
+                }
+                catch (_) { }
+                try {
+                    if (entry.stream.cancel)
+                        entry.stream.cancel();
+                }
+                catch (_) { }
+                entry.stream = null;
+            }
+            try {
+                if (entry.client && typeof entry.client.close === 'function')
+                    entry.client.close();
+            }
+            catch (_) { }
+            registry.delete(keyFor(entry.grpcUrl || '', entry.topic, entry.consumerName));
+        }
+        catch (err) {
+            // ignore
+        }
+    });
+}

package/dist/index.d.ts

@@ -1,3 +1,5 @@
 export * from './producer';
 export * from './consumer';
 export * from './config';
+export * from './message';
+export * from './consumerManager';

package/dist/index.js

@@ -17,3 +17,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./producer"), exports);
 __exportStar(require("./consumer"), exports);
 __exportStar(require("./config"), exports);
+__exportStar(require("./message"), exports);
+__exportStar(require("./consumerManager"), exports);

package/dist/message.d.ts

@@ -0,0 +1,24 @@
+export interface IMessage {
+    offset: number;
+    timestamp: number;
+    payload: any;
+    headers: Record<string, string>;
+}
+interface MessageContext {
+    stub: any;
+    topic: string;
+    consumerName: string;
+    offset: number;
+    committed?: boolean;
+}
+export declare function runWithContext(ctx: MessageContext, fn: () => void): void;
+export declare function getContext(): MessageContext | undefined;
+export declare function commit(): Promise<void>;
+export declare class Message implements IMessage {
+    offset: number;
+    timestamp: number;
+    payload: any;
+    headers: Record<string, string>;
+    constructor(protoMsg: any);
+}
+export {};

package/dist/message.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Message = void 0;
+exports.runWithContext = runWithContext;
+exports.getContext = getContext;
+exports.commit = commit;
+const async_hooks_1 = require("async_hooks");
+const storage = new async_hooks_1.AsyncLocalStorage();
+function runWithContext(ctx, fn) {
+    storage.run(ctx, fn);
+}
+function getContext() {
+    return storage.getStore();
+}
+async function commit() {
+    const ctx = storage.getStore();
+    if (!ctx)
+        throw new Error('commit() called outside of a consumer handler');
+    if (ctx.committed)
+        return;
+    return new Promise((resolve, reject) => {
+        try {
+            ctx.stub.CommitOffset({ topic: ctx.topic, consumer_name: ctx.consumerName, offset: ctx.offset + 1 }, (err, res) => {
+                if (err)
+                    return reject(err);
+                ctx.committed = true;
+                resolve();
+            });
+        }
+        catch (e) {
+            reject(e);
+        }
+    });
+}
+class Message {
+    constructor(protoMsg) {
+        this.offset = protoMsg.offset;
+        this.timestamp = protoMsg.timestamp;
+        this.headers = {};
+        try {
+            this.headers = Object.assign({}, protoMsg.headers);
+        }
+        catch (e) {
+            this.headers = {};
+        }
+        const buf = protoMsg.payload;
+        const ptype = this.headers['payload-type'];
+        if (ptype === 'json') {
+            try {
+                this.payload = JSON.parse(buf.toString());
+            }
+            catch (e) {
+                this.payload = buf;
+            }
+        }
+        else if (ptype === 'string') {
+            try {
+                this.payload = buf.toString('utf8');
+            }
+            catch (e) {
+                this.payload = buf;
+            }
+        }
+        else if (ptype === 'bytes') {
+            this.payload = buf;
+        }
+        else {
+            try {
+                this.payload = JSON.parse(buf.toString());
+            }
+            catch (e) {
+                this.payload = buf;
+            }
+        }
+    }
+}
+exports.Message = Message;

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "pulse-sdk",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "Pulse SDK for Node.js/TypeScript",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "build": "tsc",
     "postbuild": "node scripts/copy-proto.js",
-    "test": "jest"
+    "test": "npm run build && jest"
   },
   "author": "",
   "license": "MIT",
@@ -22,19 +22,21 @@
   "devDependencies": {
     "typescript": "^5.0.0",
     "jest": "^29.0.0",
-    "ts-jest": "^29.0.0",
-    "@types/jest": "^29.0.0"
+    "ts-jest": "^29.0.0"
+    ,"@types/jest": "^29.0.0",
+    "@types/js-yaml": "^4.0.5"
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.8.0",
     "protobufjs": "^7.2.0",
     "@grpc/proto-loader": "^0.7.0"
-  },
+    ,"js-yaml": "^4.1.0"
+  }
+,
   "jest": {
     "preset": "ts-jest",
     "testEnvironment": "node",
-    "testMatch": [
-      "**/tests/**/*.test.ts"
-    ]
+    "testMatch": ["**/tests/**/*.test.ts"],
+    "globalTeardown": "<rootDir>/tests/globalTeardown.js"
   }
 }