pulse-sdk 0.0.5 → 0.1.0-main.0

package/dist/consumer.js

@@ -1,124 +1,173 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Consumer = void 0;
+exports.Message = exports.commit = void 0;
+exports.stop = stop;
+exports.consumer = consumer;
+exports.run = run;
+const config_1 = require("./config");
 const client_1 = require("./proto/client");
-const consumerManager_1 = require("./consumerManager");
-const crypto_1 = require("crypto");
 const message_1 = require("./message");
-class Consumer {
-    constructor(config) {
-        this.config = config;
-        this.handlers = {};
-        this.unregisterFns = [];
-        this.client = (0, client_1.createClient)(config.grpcUrl);
+Object.defineProperty(exports, "Message", { enumerable: true, get: function () { return message_1.Message; } });
+Object.defineProperty(exports, "commit", { enumerable: true, get: function () { return message_1.commit; } });
+const crypto_1 = require("crypto");
+const _consumers = [];
+const _cleanupFunctions = [];
+function stop() {
+    _cleanupFunctions.forEach(fn => fn());
+    _cleanupFunctions.length = 0;
+    _consumers.length = 0;
+}
+function consumer(topic, handler, options = {}) {
+    const config = (0, config_1.getConfig)();
+    const host = options.host || config.broker.host;
+    const port = options.port || config.broker.grpc_port;
+    const baseGroup = options.consumerGroup || config.client.id;
+    const grouped = options.grouped !== undefined ? options.grouped : true;
+    let group = baseGroup;
+    if (!grouped) {
+        group = `${baseGroup}-${(0, crypto_1.randomUUID)().replace(/-/g, '')}`;
+    }
+    let autoCommit = options.autoCommit;
+    if (autoCommit === undefined) {
+        autoCommit = config.client.auto_commit;
+        // Check topic specific config
+        const topicCfg = config.topics?.find(t => t.name === topic);
+        // Note: Python SDK checks topic_config["consume"]["auto_commit"] but our config.ts 
+        // currently only has create_if_missing and config (fifo, retention).
+        // We'll stick to global client config for now unless we expand TopicConfig.
     }
-    on(eventType, handler) {
-        if (!this.handlers[eventType]) {
-            this.handlers[eventType] = [];
+    _consumers.push({
+        topic,
+        host,
+        port,
+        group,
+        autoCommit: autoCommit,
+        handler,
+        grouped,
+    });
+}
+async function run() {
+    // Group consumers by (topic, host, port, group)
+    const groupedMap = new Map();
+    for (const c of _consumers) {
+        const key = `${c.topic}:${c.host}:${c.port}:${c.group}`;
+        if (!groupedMap.has(key)) {
+            groupedMap.set(key, {
+                topic: c.topic,
+                host: c.host,
+                port: c.port,
+                group: c.group,
+                autoCommit: c.autoCommit,
+                handlers: [],
+            });
         }
-        this.handlers[eventType].push(handler);
+        groupedMap.get(key).handlers.push(c.handler);
+    }
+    const promises = [];
+    for (const groupConfig of groupedMap.values()) {
+        promises.push(consumeLoopGroup(groupConfig));
     }
-    async start(topic, consumerName = 'default') {
-        const topicName = topic || this.config.eventTypes[0];
-        // If grouped is enabled (default true) use shared in-process stream,
-        // otherwise create a dedicated stream (each consumer gets all messages).
-        const grouped = this.config.grouped !== false;
-        if (grouped) {
-            // register handlers for this topic to shared stream
-            const handlers = this.handlers[topicName] || [];
-            for (const h of handlers) {
-                const unregister = (0, consumerManager_1.registerSharedHandler)(this.config.grpcUrl, topicName, consumerName, (msg, stub, offset) => {
-                    // run handler with context so commit() works and support auto-commit
-                    // debug: console.log('consumer.wrapper.invoke', consumerName, topicName);
-                    (async () => {
-                        await (0, message_1.runWithContext)({ stub: stub || this.client, topic: topicName, consumerName, offset: offset ?? msg.offset }, async () => {
-                            try {
-                                const r = h(msg);
-                                if (r && typeof r.then === 'function')
-                                    await r;
-                            }
-                            catch (e) { /* ignore */ }
-                            if (this.config.autoCommit !== false) {
-                                try {
-                                    await (0, message_1.commit)();
-                                }
-                                catch (_) { /* ignore commit errors */ }
-                            }
-                        });
-                    })().catch(() => { });
-                }, { autoCommit: this.config.autoCommit });
-                this.unregisterFns.push(unregister);
+    // We don't await promises here to let them run in background, 
+    // but we could if we wanted to block until they all finish (which they won't).
+    // However, to keep the process alive, the user should probably await this or we return a promise that never resolves?
+    // Python's run() blocks. In Node, usually we just start things.
+    // But if the script ends, the process exits.
+    // We'll return a promise that never resolves to simulate blocking if awaited.
+    return new Promise(() => { });
+}
+async function consumeLoopGroup(groupConfig) {
+    const address = `${groupConfig.host}:${groupConfig.port}`;
+    // console.log(`Starting consumer for topic '${groupConfig.topic}' (group: ${groupConfig.group}) on ${address}`);
+    let handlerIdx = 0;
+    let currentStream = null;
+    let retryTimeout = null;
+    let isStopped = false;
+    const cleanup = () => {
+        isStopped = true;
+        if (currentStream) {
+            try {
+                currentStream.cancel();
             }
-            // Return a promise that never resolves (stream runs until process exits)
-            return new Promise(() => { });
-        }
-        // If grouped is explicitly false, and the consumerName equals the configured
-        // client name or the default literal, generate a unique consumer id so each
-        // consumer receives messages independently (mirrors Python behaviour).
-        if (!grouped) {
-            const base = this.config.consumerName || 'default';
-            if (consumerName === base || consumerName === 'default') {
-                consumerName = `${base}-${(0, crypto_1.randomUUID)().replace(/-/g, '')}`;
+            catch (e) {
+                // Ignore cancel errors
             }
+            currentStream = null;
         }
-        const req = { topic: topicName, consumer_name: consumerName, offset: 0 };
-        let stream = null;
-        try {
-            stream = this.client.Consume(req);
+        if (retryTimeout) {
+            clearTimeout(retryTimeout);
+            retryTimeout = null;
         }
-        catch (err) {
-            // If the client throws synchronously (e.g. CANCELLED), return a promise
-            // that is rejected but handled to avoid an unhandled rejection when
-            // callers don't await `start()` (tests call start() without awaiting).
-            const p = Promise.reject(err);
-            p.catch(() => { }); // swallow to avoid uncaught rejection
-            return p;
+    };
+    _cleanupFunctions.push(cleanup);
+    const startStream = async () => {
+        if (isStopped)
+            return;
+        try {
+            const client = (0, client_1.createClient)(address);
+            const req = {
+                topic: groupConfig.topic,
+                consumer_name: groupConfig.group,
+                offset: 0,
+            };
+            const stream = client.Consume(req);
+            currentStream = stream;
+            stream.on('data', async (protoMsg) => {
+                if (isStopped)
+                    return;
+                // Pause stream to process message sequentially
+                stream.pause();
+                const msg = new message_1.Message(protoMsg);
+                if (groupConfig.handlers.length === 0) {
+                    stream.resume();
+                    return;
+                }
+                const handler = groupConfig.handlers[handlerIdx % groupConfig.handlers.length];
+                handlerIdx++;
+                const ctx = {
+                    stub: client,
+                    topic: groupConfig.topic,
+                    consumerGroup: groupConfig.group,
+                    offset: msg.offset,
+                    committed: false,
+                };
+                try {
+                    await (0, message_1.runWithContext)(ctx, async () => {
+                        await handler(msg);
+                        if (groupConfig.autoCommit && !ctx.committed) {
+                            await (0, message_1.commit)();
+                        }
+                    });
+                }
+                catch (e) {
+                    console.error(`Error processing message: ${e}`);
+                }
+                finally {
+                    // Resume stream after processing
+                    stream.resume();
+                }
+            });
+            stream.on('error', (err) => {
+                if (isStopped)
+                    return;
+                // 1 = CANCELLED
+                if (err.code === 1)
+                    return;
+                console.error(`Connection lost for ${groupConfig.topic}: ${err.message}. Retrying in 5s...`);
+                retryTimeout = setTimeout(startStream, 5000);
+            });
+            stream.on('end', () => {
+                if (isStopped)
+                    return;
+                // console.warn(`Stream ended for ${groupConfig.topic}. Retrying in 5s...`);
+                retryTimeout = setTimeout(startStream, 5000);
+            });
         }
-        stream.on('data', (msg) => {
-            const message = new message_1.Message(msg);
-            const handlers = this.handlers[req.topic] || [];
-            for (const h of handlers) {
-                // run handler within AsyncLocalStorage context so commit() can access stub and offset
-                (async () => {
-                    try {
-                        await (0, message_1.runWithContext)({ stub: this.client, topic: req.topic, consumerName, offset: message.offset }, async () => {
-                            try {
-                                const r = h(message);
-                                if (r && typeof r.then === 'function')
-                                    await r;
-                            }
-                            catch (e) { /* handler error ignored here */ }
-                            if (this.config.autoCommit !== false) {
-                                try {
-                                    await (0, message_1.commit)();
-                                }
-                                catch (_) { /* ignore commit errors */ }
-                            }
-                        });
-                    }
-                    catch (err) {
-                        // ignore
-                    }
-                })().catch(() => { });
-            }
-        });
-        const p = new Promise((resolve, reject) => {
-            stream.on('end', () => resolve());
-            stream.on('error', (e) => reject(e));
-        });
-        // prevent unhandled rejections when callers don't await the returned promise
-        p.catch(() => { });
-        return p;
-    }
-    // unregister any shared handlers when this consumer is discarded
-    close() {
-        for (const u of this.unregisterFns) {
-            try {
-                u();
-            }
-            catch (e) { /* ignore */ }
+        catch (e) {
+            if (isStopped)
+                return;
+            console.error(`Unexpected error in consumer ${groupConfig.topic}: ${e.message}`);
+            retryTimeout = setTimeout(startStream, 5000);
         }
-        this.unregisterFns = [];
-    }
+    };
+    startStream();
 }
-exports.Consumer = Consumer;

package/dist/index.d.ts

@@ -2,4 +2,3 @@ export * from './producer';
 export * from './consumer';
 export * from './config';
 export * from './message';
-export * from './consumerManager';

package/dist/index.js

@@ -18,4 +18,3 @@ __exportStar(require("./producer"), exports);
 __exportStar(require("./consumer"), exports);
 __exportStar(require("./config"), exports);
 __exportStar(require("./message"), exports);
-__exportStar(require("./consumerManager"), exports);

package/dist/message.d.ts

@@ -1,24 +1,19 @@
-export interface IMessage {
-    offset: number;
-    timestamp: number;
-    payload: any;
-    headers: Record<string, string>;
-}
-interface MessageContext {
+export interface MessageContext {
     stub: any;
     topic: string;
-    consumerName: string;
+    consumerGroup: string;
     offset: number;
-    committed?: boolean;
+    committed: boolean;
 }
-export declare function runWithContext(ctx: MessageContext, fn: () => void): unknown;
-export declare function getContext(): MessageContext | undefined;
+export declare function runWithContext(ctx: MessageContext, fn: () => void | Promise<void>): void | Promise<void>;
 export declare function commit(): Promise<void>;
-export declare class Message implements IMessage {
+export declare class Message {
     offset: number;
     timestamp: number;
-    payload: any;
-    headers: Record<string, string>;
+    private _rawPayload;
+    private _headers;
     constructor(protoMsg: any);
+    get payload(): any;
+    get rawPayload(): Buffer;
+    get headers(): Record<string, string>;
 }
-export {};

package/dist/message.js

@@ -2,76 +2,73 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Message = void 0;
 exports.runWithContext = runWithContext;
-exports.getContext = getContext;
 exports.commit = commit;
 const async_hooks_1 = require("async_hooks");
-const storage = new async_hooks_1.AsyncLocalStorage();
+const contextStorage = new async_hooks_1.AsyncLocalStorage();
 function runWithContext(ctx, fn) {
-    return storage.run(ctx, fn);
-}
-function getContext() {
-    return storage.getStore();
+    return contextStorage.run(ctx, fn);
 }
 async function commit() {
-    const ctx = storage.getStore();
-    if (!ctx)
+    const ctx = contextStorage.getStore();
+    if (!ctx) {
         throw new Error('commit() called outside of a consumer handler');
-    if (ctx.committed)
+    }
+    if (ctx.committed) {
         return;
+    }
     return new Promise((resolve, reject) => {
-        try {
-            ctx.stub.CommitOffset({ topic: ctx.topic, consumer_name: ctx.consumerName, offset: ctx.offset + 1 }, (err, res) => {
-                if (err)
-                    return reject(err);
-                ctx.committed = true;
-                resolve();
-            });
-        }
-        catch (e) {
-            reject(e);
-        }
+        const req = {
+            topic: ctx.topic,
+            consumer_name: ctx.consumerGroup,
+            offset: ctx.offset + 1,
+        };
+        ctx.stub.CommitOffset(req, (err, res) => {
+            if (err) {
+                console.error(`Error committing offset: ${err}`);
+                return reject(err);
+            }
+            // console.log(`Committed offset ${req.offset} for ${req.consumer_name}`);
+            ctx.committed = true;
+            resolve();
+        });
     });
 }
 class Message {
     constructor(protoMsg) {
-        this.offset = protoMsg.offset;
-        this.timestamp = protoMsg.timestamp;
-        this.headers = {};
-        try {
-            this.headers = Object.assign({}, protoMsg.headers);
-        }
-        catch (e) {
-            this.headers = {};
-        }
-        const buf = protoMsg.payload;
-        const ptype = this.headers['payload-type'];
+        this.offset = typeof protoMsg.offset === 'string' ? parseInt(protoMsg.offset, 10) : protoMsg.offset;
+        this.timestamp = typeof protoMsg.timestamp === 'string' ? parseInt(protoMsg.timestamp, 10) : protoMsg.timestamp;
+        this._rawPayload = protoMsg.payload;
+        this._headers = protoMsg.headers || {};
+    }
+    get payload() {
+        const ptype = this._headers['payload-type'];
         if (ptype === 'json') {
             try {
-                this.payload = JSON.parse(buf.toString());
+                return JSON.parse(this._rawPayload.toString('utf-8'));
             }
             catch (e) {
-                this.payload = buf;
+                return this._rawPayload;
             }
         }
-        else if (ptype === 'string') {
-            try {
-                this.payload = buf.toString('utf8');
-            }
-            catch (e) {
-                this.payload = buf;
-            }
+        if (ptype === 'string') {
+            return this._rawPayload.toString('utf-8');
         }
-        else if (ptype === 'bytes') {
-            this.payload = buf;
+        if (ptype === 'bytes') {
+            return this._rawPayload;
         }
-        else {
-            try {
-                this.payload = JSON.parse(buf.toString());
-            }
-            catch (e) {
-                this.payload = buf;
-            }
+        // Fallback
+        try {
+            return JSON.parse(this._rawPayload.toString('utf-8'));
+        }
+        catch (e) {
+            return this._rawPayload;
         }
     }
+    get rawPayload() {
+        return this._rawPayload;
+    }
+    get headers() {
+        return this._headers;
+    }
 }
 exports.Message = Message;

package/dist/producer.d.ts

@@ -1,11 +1,8 @@
-import { PulseConfig } from './config';
-export interface PublishResponse {
-    id: string;
-    offset: number | string;
-}
 export declare class Producer {
-    private config;
     private client;
-    constructor(config: PulseConfig);
-    send(topic: string, payload: any, headers?: Record<string, string>): Promise<PublishResponse>;
+    private config;
+    constructor(host?: string, port?: number);
+    private setupTopics;
+    send(topic: string, payload: any): Promise<void>;
+    close(): void;
 }

package/dist/producer.js

@@ -1,25 +1,68 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Producer = void 0;
+const config_1 = require("./config");
 const client_1 = require("./proto/client");
 class Producer {
-    constructor(config) {
-        this.config = config;
-        this.client = (0, client_1.createClient)(config.grpcUrl);
+    constructor(host, port) {
+        this.config = (0, config_1.getConfig)();
+        const h = host || this.config.broker.host;
+        const p = port || this.config.broker.grpc_port;
+        const address = `${h}:${p}`;
+        this.client = (0, client_1.createClient)(address);
+        this.setupTopics();
     }
-    async send(topic, payload, headers) {
+    setupTopics() {
+        const topics = this.config.topics || [];
+        for (const topicCfg of topics) {
+            if (topicCfg.create_if_missing) {
+                const req = {
+                    topic: topicCfg.name,
+                    fifo: topicCfg.config?.fifo || false,
+                    retention_bytes: topicCfg.config?.retention_bytes || 0,
+                };
+                this.client.CreateTopic(req, (err, res) => {
+                    // Ignore errors (e.g. topic already exists)
+                    if (err) {
+                        // console.warn(`Failed to create topic ${topicCfg.name}:`, err.message);
+                    }
+                });
+            }
+        }
+    }
+    async send(topic, payload) {
+        let data;
+        const headers = {};
+        if (Buffer.isBuffer(payload)) {
+            data = payload;
+            headers['payload-type'] = 'bytes';
+        }
+        else if (typeof payload === 'string') {
+            data = Buffer.from(payload, 'utf-8');
+            headers['payload-type'] = 'string';
+        }
+        else if (typeof payload === 'object') {
+            data = Buffer.from(JSON.stringify(payload), 'utf-8');
+            headers['payload-type'] = 'json';
+        }
+        else {
+            throw new Error('Payload must be Buffer, string, or object');
+        }
         const req = {
             topic,
-            payload: Buffer.from(JSON.stringify(payload)),
-            headers: headers || {},
+            payload: data,
+            headers,
         };
         return new Promise((resolve, reject) => {
             this.client.Publish(req, (err, res) => {
                 if (err)
                     return reject(err);
-                resolve({ id: res.id, offset: res.offset });
+                resolve();
             });
         });
     }
+    close() {
+        this.client.close();
+    }
 }
 exports.Producer = Producer;

package/dist/proto/client.d.ts

@@ -1,2 +1,5 @@
+import * as protoLoader from '@grpc/proto-loader';
+export declare const packageDefinition: protoLoader.PackageDefinition;
+export declare const loaded: any;
 export type PulseClient = any;
 export declare function createClient(address: string): PulseClient;

package/dist/proto/client.js

@@ -36,28 +36,42 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.loaded = exports.packageDefinition = void 0;
 exports.createClient = createClient;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const grpc = __importStar(require("@grpc/grpc-js"));
 const protoLoader = __importStar(require("@grpc/proto-loader"));
-// Prefer the proto bundled with the package (dist/proto/pulse.proto).
-// In development (when running from source), fall back to the repository proto.
+// Prefer the cached proto (downloaded from server), then bundled, then repo.
+const cachedProto = path_1.default.resolve(__dirname, 'pulse.cached.proto');
 const bundledProto = path_1.default.resolve(__dirname, 'pulse.proto');
 const repoProto = path_1.default.resolve(__dirname, '../../../../internal/api/proto/pulse.proto');
+// Try to update proto from server
+// try {
+//   const host = process.env.PULSE_HOST || 'localhost';
+//   const port = process.env.PULSE_HTTP_PORT || '5555';
+//   const url = `http://${host}:${port}/proto`;
+//   // Use curl to download with a short timeout (1s)
+//   execSync(`curl -s -m 1 -o "${cachedProto}" "${url}"`, { stdio: 'ignore' });
+// } catch (e) {
+//   // Failed to fetch, will fall back to existing files
+// }
 let PROTO_PATH = bundledProto;
-if (!fs_1.default.existsSync(bundledProto)) {
+if (fs_1.default.existsSync(cachedProto)) {
+    PROTO_PATH = cachedProto;
+}
+else if (!fs_1.default.existsSync(bundledProto)) {
     // fallback to repo proto for local development
     PROTO_PATH = repoProto;
 }
-const packageDefinition = protoLoader.loadSync(PROTO_PATH, {
+exports.packageDefinition = protoLoader.loadSync(PROTO_PATH, {
     keepCase: true,
     longs: String,
     enums: String,
     defaults: true,
     oneofs: true,
 });
-const loaded = grpc.loadPackageDefinition(packageDefinition).pulse.v1;
+exports.loaded = grpc.loadPackageDefinition(exports.packageDefinition).pulse.v1;
 function createClient(address) {
-    return new loaded.PulseService(address, grpc.credentials.createInsecure());
+    return new exports.loaded.PulseService(address, grpc.credentials.createInsecure());
 }

package/dist/proto/pulse.proto

@@ -56,9 +56,18 @@ message CreateTopicRequest {
   string topic = 1;
   bool fifo = 2;
   int64 retention_bytes = 3;
-  int64 retention_time = 4; // Nanoseconds
+  int64 retention_time = 4;
+  int32 flush_threshold = 5;
+  int64 flush_interval = 6;
+  int64 segment_size = 7;
 }
 
 message CreateTopicResponse {
   bool success = 1;
 }
+
+message ListTopicsRequest {}
+
+message ListTopicsResponse {
+  repeated string topics = 1;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pulse-sdk",
-  "version": "0.0.5",
+  "version": "0.1.0-main.0",
   "description": "Pulse SDK for Node.js/TypeScript",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -22,21 +22,22 @@
   "devDependencies": {
     "typescript": "^5.0.0",
     "jest": "^29.0.0",
-    "ts-jest": "^29.0.0"
-    ,"@types/jest": "^29.0.0",
+    "ts-jest": "^29.0.0",
+    "@types/jest": "^29.0.0",
     "@types/js-yaml": "^4.0.5"
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.8.0",
     "protobufjs": "^7.2.0",
-    "@grpc/proto-loader": "^0.7.0"
-    ,"js-yaml": "^4.1.0"
-  }
-,
+    "@grpc/proto-loader": "^0.7.0",
+    "js-yaml": "^4.1.0"
+  },
   "jest": {
     "preset": "ts-jest",
     "testEnvironment": "node",
-    "testMatch": ["**/tests/**/*.test.ts"],
+    "testMatch": [
+      "**/tests/**/*.test.ts"
+    ],
     "globalTeardown": "<rootDir>/tests/globalTeardown.js"
   }
 }