rehydra 0.3.3 → 0.4.0

package/dist/streaming/anonymizer-stream.js

@@ -0,0 +1,184 @@
+/**
+ * Anonymizer Stream
+ * Node.js Transform stream that anonymizes text chunk-by-chunk
+ * using sentence buffering for NER context.
+ */
+import { Transform } from "node:stream";
+import { SentenceBuffer } from "./sentence-buffer.js";
+import { encryptPIIMap } from "../crypto/index.js";
+/**
+ * Transform stream that anonymizes text passing through it.
+ *
+ * Uses a SentenceBuffer to accumulate text and flush at sentence
+ * boundaries, maintaining overlap for NER context.
+ *
+ * @example
+ * ```typescript
+ * const stream = await createAnonymizerStream({
+ *   anonymizer: { ner: { mode: 'quantized' } },
+ *   sessionId: 'chat-123',
+ *   piiStorageProvider: storage,
+ *   keyProvider: keyProvider,
+ * });
+ *
+ * createReadStream('input.txt')
+ *   .pipe(stream)
+ *   .pipe(createWriteStream('anonymized.txt'));
+ * ```
+ */
+export class AnonymizerStream extends Transform {
+    sentenceBuffer;
+    keyProvider;
+    piiStorageProvider;
+    sessionId;
+    onChunkCallback;
+    onFinishCallback;
+    saveIntervalMs;
+    lastSaveTime = 0;
+    totalEntities = 0;
+    startTime = 0;
+    chunkCount = 0;
+    constructor(anonymizer, config, initialPiiMap) {
+        super({
+            ...config.streamOptions,
+            decodeStrings: true,
+            encoding: "utf-8",
+        });
+        this.keyProvider = config.keyProvider ?? null;
+        this.piiStorageProvider = config.piiStorageProvider ?? null;
+        this.sessionId = config.sessionId ?? null;
+        this.onChunkCallback = config.onChunk;
+        this.onFinishCallback = config.onFinish;
+        this.saveIntervalMs = config.saveIntervalMs ?? null;
+        this.sentenceBuffer = new SentenceBuffer(anonymizer, config.buffer, {
+            keyProvider: this.keyProvider ?? undefined,
+            locale: config.locale,
+            policy: config.policy,
+            initialPiiMap,
+        });
+    }
+    /**
+     * Get the cumulative PII map built across all chunks.
+     * Available after stream finishes.
+     */
+    getPiiMap() {
+        return this.sentenceBuffer.getCumulativePiiMap();
+    }
+    /**
+     * Get stream statistics.
+     */
+    get stats() {
+        return {
+            totalEntities: this.totalEntities,
+            chunksProcessed: this.chunkCount,
+            totalProcessingTimeMs: this.startTime > 0 ? performance.now() - this.startTime : 0,
+        };
+    }
+    _transform(chunk, encoding, callback) {
+        if (this.startTime === 0) {
+            this.startTime = performance.now();
+        }
+        const text = typeof chunk === "string" ? chunk : chunk.toString("utf-8");
+        this.processChunk(text)
+            .then(() => callback())
+            .catch((err) => callback(err));
+    }
+    _flush(callback) {
+        this.processFlush()
+            .then(() => callback())
+            .catch((err) => callback(err));
+    }
+    async processChunk(text) {
+        const chunkStart = performance.now();
+        const results = await this.sentenceBuffer.append(text);
+        for (const result of results) {
+            this.chunkCount++;
+            this.totalEntities += result.entities.length;
+            this.push(result.anonymizedText);
+            this.onChunkCallback?.({
+                anonymizedText: result.anonymizedText,
+                entities: result.entities,
+                totalEntities: this.totalEntities,
+                processingTimeMs: performance.now() - chunkStart,
+            });
+        }
+        // Debounced save: save PII map at most once per saveIntervalMs
+        if (this.saveIntervalMs !== null && results.length > 0) {
+            await this.maybeSaveToStorage();
+        }
+    }
+    async maybeSaveToStorage() {
+        if (this.sessionId === null || this.piiStorageProvider === null || this.keyProvider === null) {
+            return;
+        }
+        const now = performance.now();
+        if (now - this.lastSaveTime < (this.saveIntervalMs ?? Infinity)) {
+            return;
+        }
+        const piiMap = this.sentenceBuffer.getCumulativePiiMap();
+        if (piiMap.size === 0)
+            return;
+        const key = await this.keyProvider.getKey();
+        const encrypted = await encryptPIIMap(piiMap, key);
+        await this.piiStorageProvider.save(this.sessionId, encrypted, {
+            createdAt: Date.now(),
+            entityCounts: this.buildEntityCounts(),
+        });
+        this.lastSaveTime = now;
+    }
+    async processFlush() {
+        // Flush remaining buffer
+        const results = await this.sentenceBuffer.flush();
+        for (const result of results) {
+            this.chunkCount++;
+            this.totalEntities += result.entities.length;
+            this.push(result.anonymizedText);
+            this.onChunkCallback?.({
+                anonymizedText: result.anonymizedText,
+                entities: result.entities,
+                totalEntities: this.totalEntities,
+                processingTimeMs: performance.now() - this.startTime,
+            });
+        }
+        // Save PII map to storage if session is configured
+        if (this.sessionId !== null && this.piiStorageProvider !== null && this.keyProvider !== null) {
+            const piiMap = this.sentenceBuffer.getCumulativePiiMap();
+            if (piiMap.size > 0) {
+                const key = await this.keyProvider.getKey();
+                const encrypted = await encryptPIIMap(piiMap, key);
+                await this.piiStorageProvider.save(this.sessionId, encrypted, {
+                    createdAt: Date.now(),
+                    entityCounts: this.buildEntityCounts(),
+                });
+            }
+        }
+        // Fire finish callback
+        const totalTime = performance.now() - this.startTime;
+        let finishPiiMap;
+        if (this.keyProvider) {
+            const piiMap = this.sentenceBuffer.getCumulativePiiMap();
+            if (piiMap.size > 0) {
+                const key = await this.keyProvider.getKey();
+                finishPiiMap = await encryptPIIMap(piiMap, key);
+            }
+        }
+        this.onFinishCallback?.({
+            totalEntities: this.totalEntities,
+            piiMap: finishPiiMap,
+            totalProcessingTimeMs: totalTime,
+        });
+    }
+    buildEntityCounts() {
+        const counts = {};
+        for (const key of this.sentenceBuffer.getCumulativePiiMap().keys()) {
+            // Keys are in format "TYPE_ID" e.g. "EMAIL_1", "PERSON_2"
+            const underscoreIdx = key.lastIndexOf("_");
+            if (underscoreIdx !== -1) {
+                const type = key.slice(0, underscoreIdx);
+                counts[type] = (counts[type] ?? 0) + 1;
+            }
+        }
+        return counts;
+    }
+}
+//# sourceMappingURL=anonymizer-stream.js.map

package/dist/streaming/anonymizer-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"anonymizer-stream.js","sourceRoot":"","sources":["../../src/streaming/anonymizer-stream.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAItD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,gBAAiB,SAAQ,SAAS;IAC5B,cAAc,CAAiB;IAC/B,WAAW,CAAqB;IAChC,kBAAkB,CAA4B;IAC9C,SAAS,CAAgB;IACzB,eAAe,CAAqC;IACpD,gBAAgB,CAAsC;IAEtD,cAAc,CAAgB;IACvC,YAAY,GAAG,CAAC,CAAC;IAEjB,aAAa,GAAG,CAAC,CAAC;IAClB,SAAS,GAAG,CAAC,CAAC;IACd,UAAU,GAAG,CAAC,CAAC;IAEvB,YACE,UAAuB,EACvB,MAAoB,EACpB,aAAyB;QAEzB,KAAK,CAAC;YACJ,GAAG,MAAM,CAAC,aAAa;YACvB,aAAa,EAAE,IAAI;YACnB,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC5D,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC1C,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC;QACtC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;YAClE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;YAC1C,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa;SACd,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,IAAI,KAAK;QAKP,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,eAAe,EAAE,IAAI,CAAC,UAAU;YAChC,qBAAqB,EACnB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;SAC9D,CAAC;IACJ,CAAC;IAEQ,UAAU,CACjB,KAAsB,EACtB,QAAwB,EACxB,QAA2B;QAE3B,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAEzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;aACpB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;aACtB,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEQ,MAAM,CAAC,QAA2B;QACzC,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;aACtB,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAY;QACrC,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAEjC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,gBAAgB,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU;aACjD,CAAC,CAAC;QACL,CAAC;QAED,+DAA+D;QAC/D,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC7F,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,GAAG,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC,EAAE,CAAC;YAChE,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QACzD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAE9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE;YAC5D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,EAAE;SACvC,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAElD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAEjC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,gBAAgB,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS;aACrD,CAAC,CAAC;QACL,CAAC;QAED,mDAAmD;QACnD,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC7F,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBAC5C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBACnD,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE;oBAC5D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,EAAE;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,YAAY,CAAC;QACjB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBAC5C,YAAY,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,MAAM,EAAE,YAAY;YACpB,qBAAqB,EAAE,SAAS;SACjC,CAAC,CAAC;IACL,CAAC;IAEO,iBAAiB;QACvB,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YACnE,0DAA0D;YAC1D,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/streaming/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Streaming Anonymization Module
+ * Node.js/Bun only — not available in browser builds.
+ */
+export { AnonymizerStream } from "./anonymizer-stream.js";
+export { createAnonymizerStream } from "./stream-factory.js";
+export { SentenceBuffer, resolveBufferConfig } from "./sentence-buffer.js";
+export type { StreamConfig, SentenceBufferConfig, StreamChunkEvent, StreamFinishEvent, FlushResult, ResolvedBufferConfig, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/streaming/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/streaming/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,oBAAoB,GACrB,MAAM,YAAY,CAAC"}

package/dist/streaming/index.js

@@ -0,0 +1,8 @@
+/**
+ * Streaming Anonymization Module
+ * Node.js/Bun only — not available in browser builds.
+ */
+export { AnonymizerStream } from "./anonymizer-stream.js";
+export { createAnonymizerStream } from "./stream-factory.js";
+export { SentenceBuffer, resolveBufferConfig } from "./sentence-buffer.js";
+//# sourceMappingURL=index.js.map

package/dist/streaming/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/streaming/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/streaming/sentence-buffer.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Sentence Buffer
+ * Accumulates incoming text and flushes complete sentences for anonymization,
+ * keeping an overlap region for NER context at chunk boundaries.
+ */
+import type { AnonymizationPolicy } from "../types/index.js";
+import type { RawPIIMap } from "../pipeline/tagger.js";
+import type { KeyProvider } from "../crypto/index.js";
+import type { IAnonymizer } from "../storage/session.js";
+import type { SentenceBufferConfig, ResolvedBufferConfig, FlushResult } from "./types.js";
+/**
+ * Resolves user config with defaults
+ */
+export declare function resolveBufferConfig(config?: SentenceBufferConfig): ResolvedBufferConfig;
+/**
+ * SentenceBuffer accumulates text and flushes at sentence boundaries,
+ * maintaining an overlap region for NER context across boundaries.
+ */
+export declare class SentenceBuffer {
+    private buffer;
+    private overlapSuffix;
+    private cumulativePiiMap;
+    private totalEntities;
+    private readonly config;
+    private readonly anonymizer;
+    private readonly keyProvider;
+    private readonly locale?;
+    private readonly policy?;
+    constructor(anonymizer: IAnonymizer, config?: SentenceBufferConfig, options?: {
+        keyProvider?: KeyProvider;
+        locale?: string;
+        policy?: Partial<AnonymizationPolicy>;
+        initialPiiMap?: RawPIIMap;
+    });
+    /**
+     * Append a chunk of text. Returns flush results if sentence boundaries
+     * were found and text was anonymized, or an empty array if buffering.
+     */
+    append(chunk: string): Promise<FlushResult[]>;
+    /**
+     * Flush all remaining buffered text (called on stream end).
+     */
+    flush(): Promise<FlushResult[]>;
+    /**
+     * Get the cumulative PII map built across all flushes.
+     */
+    getCumulativePiiMap(): RawPIIMap;
+    /**
+     * Get total entity count across all flushes.
+     */
+    getTotalEntities(): number;
+    /**
+     * Find and flush all available sentence-delimited segments from the buffer.
+     */
+    private flushAvailable;
+    /**
+     * Find the last sentence boundary position in the buffer.
+     * Returns the index after the boundary (split point), or -1 if none found.
+     */
+    private findLastBoundary;
+    /**
+     * Process a segment of text: prepend overlap, anonymize, extract new output.
+     */
+    private processSegment;
+    /**
+     * Extract the new (non-overlap) portion from the anonymized text.
+     *
+     * The anonymized text covers `overlapSuffix + segment`. We need to find
+     * the split point in the anonymized text that corresponds to the original
+     * offset where the new content begins.
+     *
+     * We walk through the original text and anonymized text simultaneously,
+     * advancing past PII tags in the anonymized text while tracking the
+     * corresponding position in the original text.
+     */
+    private extractNewOutput;
+}
+//# sourceMappingURL=sentence-buffer.d.ts.map

package/dist/streaming/sentence-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-buffer.d.ts","sourceRoot":"","sources":["../../src/streaming/sentence-buffer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EACV,oBAAoB,EACpB,oBAAoB,EACpB,WAAW,EACZ,MAAM,YAAY,CAAC;AAkBpB;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,CAAC,EAAE,oBAAoB,GAC5B,oBAAoB,CAYtB;AAED;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,aAAa,CAAM;IAC3B,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAc;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA+B;gBAGrD,UAAU,EAAE,WAAW,EACvB,MAAM,CAAC,EAAE,oBAAoB,EAC7B,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACtC,aAAa,CAAC,EAAE,SAAS,CAAC;KAC3B;IAaH;;;OAGG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAYnD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAWrC;;OAEG;IACH,mBAAmB,IAAI,SAAS;IAIhC;;OAEG;IACH,gBAAgB,IAAI,MAAM;IAI1B;;OAEG;YACW,cAAc;IA8B5B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;OAEG;YACW,cAAc;IA2D5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB;CA2DzB"}

package/dist/streaming/sentence-buffer.js

@@ -0,0 +1,238 @@
+/**
+ * Sentence Buffer
+ * Accumulates incoming text and flushes complete sentences for anonymization,
+ * keeping an overlap region for NER context at chunk boundaries.
+ */
+import { decryptPIIMap } from "../crypto/index.js";
+const DEFAULT_SENTENCE_BOUNDARY = /[.!?]\s+|\n{2,}/;
+const DEFAULT_CONFIG = {
+    overlapChars: 100,
+    maxBufferSize: 8192,
+    minBufferSize: 50,
+    lowLatency: false,
+    sentenceBoundary: DEFAULT_SENTENCE_BOUNDARY,
+};
+const LOW_LATENCY_CONFIG = {
+    overlapChars: 50,
+    maxBufferSize: 512,
+    minBufferSize: 20,
+};
+/**
+ * Resolves user config with defaults
+ */
+export function resolveBufferConfig(config) {
+    const base = config?.lowLatency === true
+        ? { ...DEFAULT_CONFIG, ...LOW_LATENCY_CONFIG, lowLatency: true }
+        : { ...DEFAULT_CONFIG };
+    return {
+        overlapChars: config?.overlapChars ?? base.overlapChars,
+        maxBufferSize: config?.maxBufferSize ?? base.maxBufferSize,
+        minBufferSize: config?.minBufferSize ?? base.minBufferSize,
+        lowLatency: config?.lowLatency ?? base.lowLatency,
+        sentenceBoundary: config?.sentenceBoundary ?? base.sentenceBoundary,
+    };
+}
+/**
+ * SentenceBuffer accumulates text and flushes at sentence boundaries,
+ * maintaining an overlap region for NER context across boundaries.
+ */
+export class SentenceBuffer {
+    buffer = "";
+    overlapSuffix = "";
+    cumulativePiiMap = new Map();
+    totalEntities = 0;
+    config;
+    anonymizer;
+    keyProvider;
+    locale;
+    policy;
+    constructor(anonymizer, config, options) {
+        this.anonymizer = anonymizer;
+        this.config = resolveBufferConfig(config);
+        this.keyProvider = options?.keyProvider ?? null;
+        this.locale = options?.locale;
+        this.policy = options?.policy;
+        if (options?.initialPiiMap) {
+            this.cumulativePiiMap = new Map(options.initialPiiMap);
+        }
+    }
+    /**
+     * Append a chunk of text. Returns flush results if sentence boundaries
+     * were found and text was anonymized, or an empty array if buffering.
+     */
+    async append(chunk) {
+        if (chunk.length === 0)
+            return [];
+        this.buffer += chunk;
+        if (this.buffer.length < this.config.minBufferSize) {
+            return [];
+        }
+        return this.flushAvailable();
+    }
+    /**
+     * Flush all remaining buffered text (called on stream end).
+     */
+    async flush() {
+        if (this.buffer.length === 0) {
+            return [];
+        }
+        const result = await this.processSegment(this.buffer, true);
+        this.buffer = "";
+        this.overlapSuffix = "";
+        return result ? [result] : [];
+    }
+    /**
+     * Get the cumulative PII map built across all flushes.
+     */
+    getCumulativePiiMap() {
+        return new Map(this.cumulativePiiMap);
+    }
+    /**
+     * Get total entity count across all flushes.
+     */
+    getTotalEntities() {
+        return this.totalEntities;
+    }
+    /**
+     * Find and flush all available sentence-delimited segments from the buffer.
+     */
+    async flushAvailable() {
+        const results = [];
+        while (this.buffer.length >= this.config.minBufferSize) {
+            const boundaryIndex = this.findLastBoundary();
+            if (boundaryIndex === -1) {
+                // No boundary found
+                if (this.buffer.length > this.config.maxBufferSize) {
+                    // Force flush at maxBufferSize
+                    const segment = this.buffer.slice(0, this.config.maxBufferSize);
+                    this.buffer = this.buffer.slice(this.config.maxBufferSize);
+                    const result = await this.processSegment(segment, false);
+                    if (result)
+                        results.push(result);
+                }
+                else {
+                    // Wait for more data
+                    break;
+                }
+            }
+            else {
+                // Flush up to the boundary
+                const segment = this.buffer.slice(0, boundaryIndex);
+                this.buffer = this.buffer.slice(boundaryIndex);
+                const result = await this.processSegment(segment, false);
+                if (result)
+                    results.push(result);
+            }
+        }
+        return results;
+    }
+    /**
+     * Find the last sentence boundary position in the buffer.
+     * Returns the index after the boundary (split point), or -1 if none found.
+     */
+    findLastBoundary() {
+        const regex = new RegExp(this.config.sentenceBoundary.source, "g");
+        let lastIndex = -1;
+        let match;
+        while ((match = regex.exec(this.buffer)) !== null) {
+            // Split point is after the boundary match
+            lastIndex = match.index + match[0].length;
+        }
+        return lastIndex;
+    }
+    /**
+     * Process a segment of text: prepend overlap, anonymize, extract new output.
+     */
+    async processSegment(segment, isFinal) {
+        if (segment.length === 0)
+            return null;
+        // Build context text with overlap for NER boundary detection
+        const contextText = this.overlapSuffix + segment;
+        const overlapLength = this.overlapSuffix.length;
+        // Anonymize with cumulative PII map for ID consistency
+        const result = await this.anonymizer.anonymize(contextText, this.locale, this.policy, this.cumulativePiiMap);
+        // Decrypt the PII map from the result to merge into cumulative map
+        if (result.piiMap && this.keyProvider) {
+            const key = await this.keyProvider.getKey();
+            const decrypted = await decryptPIIMap(result.piiMap, key);
+            for (const [k, v] of decrypted) {
+                this.cumulativePiiMap.set(k, v);
+            }
+        }
+        // Extract only the new (non-overlap) portion of the anonymized text
+        const newOutput = this.extractNewOutput(contextText, result.anonymizedText, result.entities, overlapLength);
+        // Update overlap suffix for next segment
+        if (!isFinal) {
+            this.overlapSuffix = segment.slice(Math.max(0, segment.length - this.config.overlapChars));
+        }
+        // Filter entities to only those in the new portion
+        const newEntities = result.entities.filter((e) => e.start >= overlapLength);
+        // Adjust entity offsets to be relative to the segment (not contextText)
+        const adjustedEntities = newEntities.map((e) => ({
+            ...e,
+            start: e.start - overlapLength,
+            end: e.end - overlapLength,
+        }));
+        this.totalEntities += adjustedEntities.length;
+        return {
+            anonymizedText: newOutput,
+            entities: adjustedEntities,
+        };
+    }
+    /**
+     * Extract the new (non-overlap) portion from the anonymized text.
+     *
+     * The anonymized text covers `overlapSuffix + segment`. We need to find
+     * the split point in the anonymized text that corresponds to the original
+     * offset where the new content begins.
+     *
+     * We walk through the original text and anonymized text simultaneously,
+     * advancing past PII tags in the anonymized text while tracking the
+     * corresponding position in the original text.
+     */
+    extractNewOutput(originalContext, anonymizedText, entities, overlapLength) {
+        if (overlapLength === 0) {
+            return anonymizedText;
+        }
+        // Build a map of original positions to anonymized positions.
+        // Entities replace spans in the original text with PII tags of different lengths.
+        // We need to find where overlapLength in original space maps to in anonymized space.
+        // Sort entities by start position
+        const sortedEntities = [...entities].sort((a, b) => a.start - b.start);
+        // Walk through, tracking offset shift caused by tag replacements
+        let originalPos = 0;
+        let anonymizedPos = 0;
+        for (const entity of sortedEntities) {
+            if (entity.start >= overlapLength) {
+                // This entity is fully in the new portion — stop here
+                break;
+            }
+            // Advance to this entity's start
+            const gap = entity.start - originalPos;
+            anonymizedPos += gap;
+            originalPos = entity.start;
+            // The entity's original span length
+            const originalSpanLength = entity.end - entity.start;
+            // Find the tag in the anonymized text at this position
+            // Tags look like: <PII type="TYPE" id="N"/>
+            const tagStart = anonymizedPos;
+            const tagEnd = anonymizedText.indexOf("/>", tagStart);
+            const tagLength = tagEnd !== -1 ? tagEnd + 2 - tagStart : originalSpanLength;
+            if (entity.end <= overlapLength) {
+                // Entity is fully within the overlap — skip entirely
+                originalPos = entity.end;
+                anonymizedPos = tagStart + tagLength;
+            }
+            else {
+                // Entity spans the boundary — it belongs to overlap (already emitted)
+                originalPos = entity.end;
+                anonymizedPos = tagStart + tagLength;
+            }
+        }
+        // Advance remaining gap to the overlap boundary
+        const remainingGap = overlapLength - originalPos;
+        anonymizedPos += remainingGap;
+        return anonymizedText.slice(anonymizedPos);
+    }
+}
+//# sourceMappingURL=sentence-buffer.js.map

package/dist/streaming/sentence-buffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-buffer.js","sourceRoot":"","sources":["../../src/streaming/sentence-buffer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAQnD,MAAM,yBAAyB,GAAG,iBAAiB,CAAC;AAEpD,MAAM,cAAc,GAAyB;IAC3C,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,EAAE;IACjB,UAAU,EAAE,KAAK;IACjB,gBAAgB,EAAE,yBAAyB;CAC5C,CAAC;AAEF,MAAM,kBAAkB,GAAkC;IACxD,YAAY,EAAE,EAAE;IAChB,aAAa,EAAE,GAAG;IAClB,aAAa,EAAE,EAAE;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAA6B;IAE7B,MAAM,IAAI,GAAG,MAAM,EAAE,UAAU,KAAK,IAAI;QACtC,CAAC,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,kBAAkB,EAAE,UAAU,EAAE,IAAI,EAAE;QAChE,CAAC,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;IAE1B,OAAO;QACL,YAAY,EAAE,MAAM,EAAE,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,IAAI,CAAC,aAAa;QAC1D,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,IAAI,CAAC,aAAa;QAC1D,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,IAAI,CAAC,UAAU;QACjD,gBAAgB,EAAE,MAAM,EAAE,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;KACpE,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,GAAG,EAAE,CAAC;IACZ,aAAa,GAAG,EAAE,CAAC;IACnB,gBAAgB,GAAc,IAAI,GAAG,EAAE,CAAC;IACxC,aAAa,GAAG,CAAC,CAAC;IAET,MAAM,CAAuB;IAC7B,UAAU,CAAc;IACxB,WAAW,CAAqB;IAChC,MAAM,CAAU;IAChB,MAAM,CAAgC;IAEvD,YACE,UAAuB,EACvB,MAA6B,EAC7B,OAKC;QAED,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,IAAI,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAE9B,IAAI,OAAO,EAAE,aAAa,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACnD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,MAAM,OAAO,GAAkB,EAAE,CAAC;QAElC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACvD,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAE9C,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzB,oBAAoB;gBACpB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;oBACnD,+BAA+B;oBAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAChE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC3D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACzD,IAAI,MAAM;wBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,qBAAqB;oBACrB,MAAM;gBACR,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,2BAA2B;gBAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;gBACpD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACzD,IAAI,MAAM;oBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,gBAAgB;QACtB,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnE,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QAEnB,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAClD,0CAA0C;YAC1C,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAC1B,OAAe,EACf,OAAgB;QAEhB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QACjD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;QAEhD,uDAAuD;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC5C,WAAW,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,gBAAgB,CACtB,CAAC;QAEF,mEAAmE;QACnE,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC1D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC/B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CACrC,WAAW,EACX,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,QAAQ,EACf,aAAa,CACd,CAAC;QAEF,yCAAyC;QACzC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAChC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CACvD,CAAC;QACJ,CAAC;QAED,mDAAmD;QACnD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,aAAa,CAAC,CAAC;QAC5E,wEAAwE;QACxE,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/C,GAAG,CAAC;YACJ,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,aAAa;YAC9B,GAAG,EAAE,CAAC,CAAC,GAAG,GAAG,aAAa;SAC3B,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,aAAa,IAAI,gBAAgB,CAAC,MAAM,CAAC;QAE9C,OAAO;YACL,cAAc,EAAE,SAAS;YACzB,QAAQ,EAAE,gBAAgB;SAC3B,CAAC;IACJ,CAAC;IAED;;;;;;;;;;OAUG;IACK,gBAAgB,CACtB,eAAuB,EACvB,cAAsB,EACtB,QAAwE,EACxE,aAAqB;QAErB,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,6DAA6D;QAC7D,kFAAkF;QAClF,qFAAqF;QAErF,kCAAkC;QAClC,MAAM,cAAc,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEvE,iEAAiE;QACjE,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,IAAI,MAAM,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,sDAAsD;gBACtD,MAAM;YACR,CAAC;YAED,iCAAiC;YACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,GAAG,WAAW,CAAC;YACvC,aAAa,IAAI,GAAG,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;YAE3B,oCAAoC;YACpC,MAAM,kBAAkB,GAAG,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC;YAErD,uDAAuD;YACvD,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC;YAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACtD,MAAM,SAAS,GACb,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC;YAE7D,IAAI,MAAM,CAAC,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,qDAAqD;gBACrD,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC;gBACzB,aAAa,GAAG,QAAQ,GAAG,SAAS,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC;gBACzB,aAAa,GAAG,QAAQ,GAAG,SAAS,CAAC;YACvC,CAAC;QACH,CAAC;QAED,gDAAgD;QAChD,MAAM,YAAY,GAAG,aAAa,GAAG,WAAW,CAAC;QACjD,aAAa,IAAI,YAAY,CAAC;QAE9B,OAAO,cAAc,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7C,CAAC;CACF"}

package/dist/streaming/stream-factory.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Stream Factory
+ * Creates configured AnonymizerStream instances
+ */
+import { AnonymizerStream } from "./anonymizer-stream.js";
+import type { StreamConfig } from "./types.js";
+/**
+ * Creates a streaming anonymizer that processes text chunk-by-chunk.
+ *
+ * @example
+ * ```typescript
+ * // Basic streaming with NER
+ * const stream = await createAnonymizerStream({
+ *   anonymizer: { ner: { mode: 'quantized' } },
+ * });
+ *
+ * createReadStream('input.txt')
+ *   .pipe(stream)
+ *   .pipe(createWriteStream('anonymized.txt'));
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Low-latency mode for LLM token streams
+ * const stream = await createAnonymizerStream({
+ *   buffer: { lowLatency: true },
+ *   sessionId: 'chat-123',
+ *   piiStorageProvider: storage,
+ *   keyProvider: keyProvider,
+ * });
+ *
+ * llmTokenStream.pipe(stream).on('data', (chunk) => {
+ *   ws.send(chunk.toString());
+ * });
+ * ```
+ */
+export declare function createAnonymizerStream(config?: StreamConfig): Promise<AnonymizerStream>;
+//# sourceMappingURL=stream-factory.d.ts.map

package/dist/streaming/stream-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-factory.d.ts","sourceRoot":"","sources":["../../src/streaming/stream-factory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,GAAE,YAAiB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAkC3B"}

package/dist/streaming/stream-factory.js

@@ -0,0 +1,69 @@
+/**
+ * Stream Factory
+ * Creates configured AnonymizerStream instances
+ */
+import { createAnonymizer, } from "../core/anonymizer.js";
+import { decryptPIIMap } from "../crypto/index.js";
+import { AnonymizerStream } from "./anonymizer-stream.js";
+/**
+ * Creates a streaming anonymizer that processes text chunk-by-chunk.
+ *
+ * @example
+ * ```typescript
+ * // Basic streaming with NER
+ * const stream = await createAnonymizerStream({
+ *   anonymizer: { ner: { mode: 'quantized' } },
+ * });
+ *
+ * createReadStream('input.txt')
+ *   .pipe(stream)
+ *   .pipe(createWriteStream('anonymized.txt'));
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Low-latency mode for LLM token streams
+ * const stream = await createAnonymizerStream({
+ *   buffer: { lowLatency: true },
+ *   sessionId: 'chat-123',
+ *   piiStorageProvider: storage,
+ *   keyProvider: keyProvider,
+ * });
+ *
+ * llmTokenStream.pipe(stream).on('data', (chunk) => {
+ *   ws.send(chunk.toString());
+ * });
+ * ```
+ */
+export async function createAnonymizerStream(config = {}) {
+    // In low-latency mode, force NER disabled for minimal latency
+    let anonymizerConfig = config.anonymizer;
+    if (config.buffer?.lowLatency === true) {
+        anonymizerConfig = {
+            ...anonymizerConfig,
+            ner: { mode: "disabled" },
+        };
+    }
+    // Ensure the anonymizer uses the same key provider as the stream
+    // so that PII maps encrypted by the anonymizer can be decrypted by the stream.
+    if (config.keyProvider) {
+        anonymizerConfig = {
+            ...anonymizerConfig,
+            keyProvider: config.keyProvider,
+        };
+    }
+    // Create and initialize the anonymizer
+    const anonymizer = createAnonymizer(anonymizerConfig);
+    await anonymizer.initialize();
+    // Load existing PII map from session storage if available
+    let initialPiiMap;
+    if (config.sessionId !== undefined && config.sessionId !== "" && config.piiStorageProvider !== undefined && config.keyProvider !== undefined) {
+        const existing = await config.piiStorageProvider.load(config.sessionId);
+        if (existing !== null) {
+            const key = await config.keyProvider.getKey();
+            initialPiiMap = await decryptPIIMap(existing.piiMap, key);
+        }
+    }
+    return new AnonymizerStream(anonymizer, config, initialPiiMap);
+}
+//# sourceMappingURL=stream-factory.js.map