rehydra 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +173 -873
- package/dist/core/anonymizer.d.ts +9 -1
- package/dist/core/anonymizer.d.ts.map +1 -1
- package/dist/core/anonymizer.js +29 -7
- package/dist/core/anonymizer.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/proxy/index.d.ts +12 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +11 -0
- package/dist/proxy/index.js.map +1 -0
- package/dist/proxy/providers/anthropic.d.ts +17 -0
- package/dist/proxy/providers/anthropic.d.ts.map +1 -0
- package/dist/proxy/providers/anthropic.js +117 -0
- package/dist/proxy/providers/anthropic.js.map +1 -0
- package/dist/proxy/providers/index.d.ts +19 -0
- package/dist/proxy/providers/index.d.ts.map +1 -0
- package/dist/proxy/providers/index.js +40 -0
- package/dist/proxy/providers/index.js.map +1 -0
- package/dist/proxy/providers/openai.d.ts +17 -0
- package/dist/proxy/providers/openai.d.ts.map +1 -0
- package/dist/proxy/providers/openai.js +92 -0
- package/dist/proxy/providers/openai.js.map +1 -0
- package/dist/proxy/providers/types.d.ts +29 -0
- package/dist/proxy/providers/types.d.ts.map +1 -0
- package/dist/proxy/providers/types.js +6 -0
- package/dist/proxy/providers/types.js.map +1 -0
- package/dist/proxy/proxy-server.d.ts +53 -0
- package/dist/proxy/proxy-server.d.ts.map +1 -0
- package/dist/proxy/proxy-server.js +146 -0
- package/dist/proxy/proxy-server.js.map +1 -0
- package/dist/proxy/rehydra-fetch.d.ts +35 -0
- package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
- package/dist/proxy/rehydra-fetch.js +217 -0
- package/dist/proxy/rehydra-fetch.js.map +1 -0
- package/dist/proxy/rehydra-proxy.d.ts +40 -0
- package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
- package/dist/proxy/rehydra-proxy.js +82 -0
- package/dist/proxy/rehydra-proxy.js.map +1 -0
- package/dist/proxy/sse-parser.d.ts +59 -0
- package/dist/proxy/sse-parser.d.ts.map +1 -0
- package/dist/proxy/sse-parser.js +112 -0
- package/dist/proxy/sse-parser.js.map +1 -0
- package/dist/proxy/types.d.ts +49 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +5 -0
- package/dist/proxy/types.js.map +1 -0
- package/dist/proxy/wrap-client.d.ts +47 -0
- package/dist/proxy/wrap-client.d.ts.map +1 -0
- package/dist/proxy/wrap-client.js +70 -0
- package/dist/proxy/wrap-client.js.map +1 -0
- package/dist/storage/session.d.ts +3 -0
- package/dist/storage/session.d.ts.map +1 -1
- package/dist/storage/session.js +24 -1
- package/dist/storage/session.js.map +1 -1
- package/dist/storage/types.d.ts +16 -0
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/streaming/anonymizer-stream.d.ts +63 -0
- package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
- package/dist/streaming/anonymizer-stream.js +184 -0
- package/dist/streaming/anonymizer-stream.js.map +1 -0
- package/dist/streaming/index.d.ts +9 -0
- package/dist/streaming/index.d.ts.map +1 -0
- package/dist/streaming/index.js +8 -0
- package/dist/streaming/index.js.map +1 -0
- package/dist/streaming/sentence-buffer.d.ts +78 -0
- package/dist/streaming/sentence-buffer.d.ts.map +1 -0
- package/dist/streaming/sentence-buffer.js +238 -0
- package/dist/streaming/sentence-buffer.js.map +1 -0
- package/dist/streaming/stream-factory.d.ts +38 -0
- package/dist/streaming/stream-factory.d.ts.map +1 -0
- package/dist/streaming/stream-factory.js +69 -0
- package/dist/streaming/stream-factory.js.map +1 -0
- package/dist/streaming/types.d.ts +121 -0
- package/dist/streaming/types.d.ts.map +1 -0
- package/dist/streaming/types.js +5 -0
- package/dist/streaming/types.js.map +1 -0
- package/dist/types/index.d.ts +8 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js.map +1 -1
- package/package.json +19 -2
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-Sent Events (SSE) Parser
|
|
3
|
+
* Parses SSE protocol streams, handling chunk boundaries correctly.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* A parsed SSE event
|
|
7
|
+
*/
|
|
8
|
+
export interface SSEEvent {
|
|
9
|
+
/** Event type (defaults to "message") */
|
|
10
|
+
event: string;
|
|
11
|
+
/** Event data (concatenated from all data: lines) */
|
|
12
|
+
data: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Parses a Server-Sent Events stream chunk by chunk.
|
|
16
|
+
*
|
|
17
|
+
* SSE protocol format:
|
|
18
|
+
* ```
|
|
19
|
+
* event: message_start
|
|
20
|
+
* data: {"type": "content"}
|
|
21
|
+
*
|
|
22
|
+
* data: {"text": "Hello"}
|
|
23
|
+
*
|
|
24
|
+
* data: [DONE]
|
|
25
|
+
*
|
|
26
|
+
* ```
|
|
27
|
+
*
|
|
28
|
+
* Each event is one or more field lines terminated by a blank line.
|
|
29
|
+
*/
|
|
30
|
+
export declare class SSEParser {
|
|
31
|
+
private buffer;
|
|
32
|
+
/**
|
|
33
|
+
* Feed a chunk of text from the SSE stream.
|
|
34
|
+
* Returns zero or more fully parsed events.
|
|
35
|
+
*/
|
|
36
|
+
parse(chunk: string): SSEEvent[];
|
|
37
|
+
/**
|
|
38
|
+
* Flush remaining buffer content (call on stream end).
|
|
39
|
+
* Returns any remaining events.
|
|
40
|
+
*/
|
|
41
|
+
flush(): SSEEvent[];
|
|
42
|
+
/**
|
|
43
|
+
* Reset the parser state.
|
|
44
|
+
*/
|
|
45
|
+
reset(): void;
|
|
46
|
+
/**
|
|
47
|
+
* Parse a single raw SSE event block into an SSEEvent.
|
|
48
|
+
*/
|
|
49
|
+
private parseEvent;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Check if a string is the SSE stream termination sentinel.
|
|
53
|
+
*/
|
|
54
|
+
export declare function isSSEDone(data: string): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Serialize an SSE event back to wire format.
|
|
57
|
+
*/
|
|
58
|
+
export declare function serializeSSEEvent(event: SSEEvent): string;
|
|
59
|
+
//# sourceMappingURL=sse-parser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sse-parser.d.ts","sourceRoot":"","sources":["../../src/proxy/sse-parser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAM;IAEpB;;;OAGG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE;IAoBhC;;;OAGG;IACH,KAAK,IAAI,QAAQ,EAAE;IAUnB;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,OAAO,CAAC,UAAU;CAuBnB;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAWzD"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-Sent Events (SSE) Parser
|
|
3
|
+
* Parses SSE protocol streams, handling chunk boundaries correctly.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Parses a Server-Sent Events stream chunk by chunk.
|
|
7
|
+
*
|
|
8
|
+
* SSE protocol format:
|
|
9
|
+
* ```
|
|
10
|
+
* event: message_start
|
|
11
|
+
* data: {"type": "content"}
|
|
12
|
+
*
|
|
13
|
+
* data: {"text": "Hello"}
|
|
14
|
+
*
|
|
15
|
+
* data: [DONE]
|
|
16
|
+
*
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* Each event is one or more field lines terminated by a blank line.
|
|
20
|
+
*/
|
|
21
|
+
export class SSEParser {
|
|
22
|
+
buffer = "";
|
|
23
|
+
/**
|
|
24
|
+
* Feed a chunk of text from the SSE stream.
|
|
25
|
+
* Returns zero or more fully parsed events.
|
|
26
|
+
*/
|
|
27
|
+
parse(chunk) {
|
|
28
|
+
this.buffer += chunk;
|
|
29
|
+
const events = [];
|
|
30
|
+
// Split on double newline (event boundary)
|
|
31
|
+
// Handle both \n\n and \r\n\r\n
|
|
32
|
+
const parts = this.buffer.split(/\n\n|\r\n\r\n/);
|
|
33
|
+
// Last part may be incomplete — keep it in the buffer
|
|
34
|
+
this.buffer = parts.pop() ?? "";
|
|
35
|
+
for (const part of parts) {
|
|
36
|
+
if (part.trim() === "")
|
|
37
|
+
continue;
|
|
38
|
+
const event = this.parseEvent(part);
|
|
39
|
+
if (event)
|
|
40
|
+
events.push(event);
|
|
41
|
+
}
|
|
42
|
+
return events;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Flush remaining buffer content (call on stream end).
|
|
46
|
+
* Returns any remaining events.
|
|
47
|
+
*/
|
|
48
|
+
flush() {
|
|
49
|
+
if (this.buffer.trim() === "") {
|
|
50
|
+
this.buffer = "";
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
const event = this.parseEvent(this.buffer);
|
|
54
|
+
this.buffer = "";
|
|
55
|
+
return event ? [event] : [];
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Reset the parser state.
|
|
59
|
+
*/
|
|
60
|
+
reset() {
|
|
61
|
+
this.buffer = "";
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Parse a single raw SSE event block into an SSEEvent.
|
|
65
|
+
*/
|
|
66
|
+
parseEvent(raw) {
|
|
67
|
+
let data = "";
|
|
68
|
+
let eventType = "message";
|
|
69
|
+
for (const line of raw.split(/\n|\r\n/)) {
|
|
70
|
+
if (line.startsWith("data: ")) {
|
|
71
|
+
// Concatenate multiple data: lines with newlines
|
|
72
|
+
data += (data ? "\n" : "") + line.slice(6);
|
|
73
|
+
}
|
|
74
|
+
else if (line === "data:") {
|
|
75
|
+
// Empty data line
|
|
76
|
+
data += (data ? "\n" : "");
|
|
77
|
+
}
|
|
78
|
+
else if (line.startsWith("event: ")) {
|
|
79
|
+
eventType = line.slice(7).trim();
|
|
80
|
+
}
|
|
81
|
+
else if (line.startsWith("event:")) {
|
|
82
|
+
eventType = line.slice(6).trim();
|
|
83
|
+
}
|
|
84
|
+
// Ignore 'id:', 'retry:', and comments (lines starting with ':')
|
|
85
|
+
}
|
|
86
|
+
if (data === "" && !raw.includes("data:"))
|
|
87
|
+
return null;
|
|
88
|
+
return { event: eventType, data };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Check if a string is the SSE stream termination sentinel.
|
|
93
|
+
*/
|
|
94
|
+
export function isSSEDone(data) {
|
|
95
|
+
return data.trim() === "[DONE]";
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Serialize an SSE event back to wire format.
|
|
99
|
+
*/
|
|
100
|
+
export function serializeSSEEvent(event) {
|
|
101
|
+
let result = "";
|
|
102
|
+
if (event.event !== "message") {
|
|
103
|
+
result += `event: ${event.event}\n`;
|
|
104
|
+
}
|
|
105
|
+
// Split data on newlines for multi-line data fields
|
|
106
|
+
for (const line of event.data.split("\n")) {
|
|
107
|
+
result += `data: ${line}\n`;
|
|
108
|
+
}
|
|
109
|
+
result += "\n";
|
|
110
|
+
return result;
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=sse-parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sse-parser.js","sourceRoot":"","sources":["../../src/proxy/sse-parser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAYH;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,SAAS;IACZ,MAAM,GAAG,EAAE,CAAC;IAEpB;;;OAGG;IACH,KAAK,CAAC,KAAa;QACjB,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACrB,MAAM,MAAM,GAAe,EAAE,CAAC;QAE9B,2CAA2C;QAC3C,gCAAgC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEjD,sDAAsD;QACtD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACjB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW;QAC5B,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,SAAS,GAAG,SAAS,CAAC;QAE1B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,iDAAiD;gBACjD,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7C,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,kBAAkB;gBAClB,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBACtC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;YACD,iEAAiE;QACnE,CAAC;QAED,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACpC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,QAAQ,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAe;IAC/C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,UAAU,KAAK,CAAC,KAAK,IAAI,CAAC;IACtC,CAAC;IACD,oDAAoD;IACpD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,SAAS,IAAI,IAAI,CAAC;IAC9B,CAAC;IACD,MAAM,IAAI,IAAI,CAAC;IACf,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy Middleware Types
|
|
3
|
+
*/
|
|
4
|
+
import type { AnonymizerConfig } from "../core/anonymizer.js";
|
|
5
|
+
import type { AnonymizationPolicy } from "../types/index.js";
|
|
6
|
+
import type { KeyProvider } from "../crypto/index.js";
|
|
7
|
+
import type { PIIStorageProvider } from "../storage/types.js";
|
|
8
|
+
/**
|
|
9
|
+
* Configuration for the Rehydra fetch wrapper
|
|
10
|
+
*/
|
|
11
|
+
export interface RehydraFetchConfig {
|
|
12
|
+
/** Anonymizer configuration */
|
|
13
|
+
anonymizer?: AnonymizerConfig;
|
|
14
|
+
/** Policy override */
|
|
15
|
+
policy?: Partial<AnonymizationPolicy>;
|
|
16
|
+
/** Key provider for encryption (required) */
|
|
17
|
+
keyProvider: KeyProvider;
|
|
18
|
+
/** Storage provider for PII map persistence (required) */
|
|
19
|
+
piiStorageProvider: PIIStorageProvider;
|
|
20
|
+
/** LLM provider name or 'auto' for auto-detection */
|
|
21
|
+
provider?: "openai" | "anthropic" | "auto";
|
|
22
|
+
/**
|
|
23
|
+
* Function to derive session ID from a request.
|
|
24
|
+
* Defaults to a UUID per request.
|
|
25
|
+
*/
|
|
26
|
+
getSessionId?: (request: Request) => string | Promise<string>;
|
|
27
|
+
/**
|
|
28
|
+
* Whether to handle streaming SSE responses.
|
|
29
|
+
* @default true
|
|
30
|
+
*/
|
|
31
|
+
handleStreaming?: boolean;
|
|
32
|
+
/** Locale hint for anonymization */
|
|
33
|
+
locale?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Configuration for the Rehydra proxy middleware
|
|
37
|
+
*/
|
|
38
|
+
export interface RehydraProxyConfig extends RehydraFetchConfig {
|
|
39
|
+
/** Upstream LLM API base URL */
|
|
40
|
+
upstream: string;
|
|
41
|
+
/**
|
|
42
|
+
* Headers to forward to upstream.
|
|
43
|
+
* @default ["authorization", "content-type", "x-api-key", "anthropic-version"]
|
|
44
|
+
*/
|
|
45
|
+
forwardHeaders?: string[];
|
|
46
|
+
/** Path prefix to strip before forwarding to upstream */
|
|
47
|
+
stripPrefix?: string;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/proxy/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,+BAA+B;IAC/B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAE9B,sBAAsB;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAEtC,6CAA6C;IAC7C,WAAW,EAAE,WAAW,CAAC;IAEzB,0DAA0D;IAC1D,kBAAkB,EAAE,kBAAkB,CAAC;IAEvC,qDAAqD;IACrD,QAAQ,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC;IAE3C;;;OAGG;IACH,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9D;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,oCAAoC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,kBAAkB;IAC5D,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAE1B,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/proxy/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM Client Wrapper
|
|
3
|
+
* Convenience function to wrap OpenAI/Anthropic SDK clients
|
|
4
|
+
* with automatic PII anonymization and rehydration.
|
|
5
|
+
*/
|
|
6
|
+
import type { RehydraFetchConfig } from "./types.js";
|
|
7
|
+
/**
|
|
8
|
+
* Wraps an LLM client (OpenAI, Anthropic, or any SDK that accepts a custom fetch)
|
|
9
|
+
* to automatically anonymize outgoing PII and rehydrate responses.
|
|
10
|
+
*
|
|
11
|
+
* Works by replacing the client's internal fetch with a Rehydra-wrapped version.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* import OpenAI from 'openai';
|
|
16
|
+
* import { wrapLLMClient, InMemoryKeyProvider, InMemoryPIIStorageProvider } from 'rehydra';
|
|
17
|
+
*
|
|
18
|
+
* const openai = wrapLLMClient(
|
|
19
|
+
* new OpenAI(),
|
|
20
|
+
* {
|
|
21
|
+
* keyProvider: new InMemoryKeyProvider(),
|
|
22
|
+
* piiStorageProvider: new InMemoryPIIStorageProvider(),
|
|
23
|
+
* },
|
|
24
|
+
* );
|
|
25
|
+
*
|
|
26
|
+
* // PII is now automatically anonymized/rehydrated
|
|
27
|
+
* const response = await openai.chat.completions.create({ ... });
|
|
28
|
+
* ```
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```typescript
|
|
32
|
+
* import Anthropic from '@anthropic-ai/sdk';
|
|
33
|
+
* import { wrapLLMClient, ConfigKeyProvider, SQLitePIIStorageProvider } from 'rehydra';
|
|
34
|
+
*
|
|
35
|
+
* const anthropic = wrapLLMClient(
|
|
36
|
+
* new Anthropic(),
|
|
37
|
+
* {
|
|
38
|
+
* keyProvider: new ConfigKeyProvider(process.env.PII_KEY!),
|
|
39
|
+
* piiStorageProvider: new SQLitePIIStorageProvider('pii.db'),
|
|
40
|
+
* },
|
|
41
|
+
* );
|
|
42
|
+
*
|
|
43
|
+
* const message = await anthropic.messages.create({ ... });
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export declare function wrapLLMClient<T>(client: T, config: Omit<RehydraFetchConfig, "provider">): T;
|
|
47
|
+
//# sourceMappingURL=wrap-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wrap-client.d.ts","sourceRoot":"","sources":["../../src/proxy/wrap-client.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAC7B,MAAM,EAAE,CAAC,EACT,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GAC3C,CAAC,CAwBH"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM Client Wrapper
|
|
3
|
+
* Convenience function to wrap OpenAI/Anthropic SDK clients
|
|
4
|
+
* with automatic PII anonymization and rehydration.
|
|
5
|
+
*/
|
|
6
|
+
import { createRehydraFetch } from "./rehydra-fetch.js";
|
|
7
|
+
/**
|
|
8
|
+
* Wraps an LLM client (OpenAI, Anthropic, or any SDK that accepts a custom fetch)
|
|
9
|
+
* to automatically anonymize outgoing PII and rehydrate responses.
|
|
10
|
+
*
|
|
11
|
+
* Works by replacing the client's internal fetch with a Rehydra-wrapped version.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* import OpenAI from 'openai';
|
|
16
|
+
* import { wrapLLMClient, InMemoryKeyProvider, InMemoryPIIStorageProvider } from 'rehydra';
|
|
17
|
+
*
|
|
18
|
+
* const openai = wrapLLMClient(
|
|
19
|
+
* new OpenAI(),
|
|
20
|
+
* {
|
|
21
|
+
* keyProvider: new InMemoryKeyProvider(),
|
|
22
|
+
* piiStorageProvider: new InMemoryPIIStorageProvider(),
|
|
23
|
+
* },
|
|
24
|
+
* );
|
|
25
|
+
*
|
|
26
|
+
* // PII is now automatically anonymized/rehydrated
|
|
27
|
+
* const response = await openai.chat.completions.create({ ... });
|
|
28
|
+
* ```
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```typescript
|
|
32
|
+
* import Anthropic from '@anthropic-ai/sdk';
|
|
33
|
+
* import { wrapLLMClient, ConfigKeyProvider, SQLitePIIStorageProvider } from 'rehydra';
|
|
34
|
+
*
|
|
35
|
+
* const anthropic = wrapLLMClient(
|
|
36
|
+
* new Anthropic(),
|
|
37
|
+
* {
|
|
38
|
+
* keyProvider: new ConfigKeyProvider(process.env.PII_KEY!),
|
|
39
|
+
* piiStorageProvider: new SQLitePIIStorageProvider('pii.db'),
|
|
40
|
+
* },
|
|
41
|
+
* );
|
|
42
|
+
*
|
|
43
|
+
* const message = await anthropic.messages.create({ ... });
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export function wrapLLMClient(client, config) {
|
|
47
|
+
const rehydraFetch = createRehydraFetch({
|
|
48
|
+
...config,
|
|
49
|
+
provider: "auto",
|
|
50
|
+
});
|
|
51
|
+
// Both OpenAI and Anthropic SDKs store fetch as a property.
|
|
52
|
+
// OpenAI uses `_fetch`, Anthropic uses `fetch`.
|
|
53
|
+
// We try both, and also support any client with a `fetch` property.
|
|
54
|
+
const clientObj = client;
|
|
55
|
+
if ("_fetch" in clientObj) {
|
|
56
|
+
// OpenAI SDK pattern
|
|
57
|
+
clientObj._fetch = rehydraFetch;
|
|
58
|
+
}
|
|
59
|
+
else if ("fetch" in clientObj) {
|
|
60
|
+
// Anthropic SDK pattern (or generic)
|
|
61
|
+
clientObj.fetch = rehydraFetch;
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
// Fallback: set both common patterns
|
|
65
|
+
clientObj._fetch = rehydraFetch;
|
|
66
|
+
clientObj.fetch = rehydraFetch;
|
|
67
|
+
}
|
|
68
|
+
return client;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=wrap-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wrap-client.js","sourceRoot":"","sources":["../../src/proxy/wrap-client.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAGxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAS,EACT,MAA4C;IAE5C,MAAM,YAAY,GAAG,kBAAkB,CAAC;QACtC,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM;KACjB,CAAC,CAAC;IAEH,4DAA4D;IAC5D,gDAAgD;IAChD,oEAAoE;IACpE,MAAM,SAAS,GAAG,MAAiC,CAAC;IAEpD,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC1B,qBAAqB;QACrB,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;IAClC,CAAC;SAAM,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;QAChC,qCAAqC;QACrC,SAAS,CAAC,KAAK,GAAG,YAAY,CAAC;IACjC,CAAC;SAAM,CAAC;QACN,qCAAqC;QACrC,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;QAChC,SAAS,CAAC,KAAK,GAAG,YAAY,CAAC;IACjC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -6,6 +6,8 @@ import type { AnonymizationResult, AnonymizationPolicy } from "../types/index.js
|
|
|
6
6
|
import type { KeyProvider } from "../crypto/index.js";
|
|
7
7
|
import type { RawPIIMap } from "../pipeline/tagger.js";
|
|
8
8
|
import type { AnonymizerSession, PIIStorageProvider, StoredPIIMap } from "./types.js";
|
|
9
|
+
import { AnonymizerStream } from "../streaming/anonymizer-stream.js";
|
|
10
|
+
import type { StreamConfig } from "../streaming/types.js";
|
|
9
11
|
/**
|
|
10
12
|
* Interface for the parent Anonymizer (to avoid circular dependency)
|
|
11
13
|
*/
|
|
@@ -26,5 +28,6 @@ export declare class AnonymizerSessionImpl implements AnonymizerSession {
|
|
|
26
28
|
load(): Promise<StoredPIIMap | null>;
|
|
27
29
|
delete(): Promise<boolean>;
|
|
28
30
|
exists(): Promise<boolean>;
|
|
31
|
+
createStream(config?: Partial<StreamConfig>): Promise<AnonymizerStream>;
|
|
29
32
|
}
|
|
30
33
|
//# sourceMappingURL=session.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/storage/session.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACb,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/storage/session.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACb,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACrC,cAAc,CAAC,EAAE,SAAS,GACzB,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACjC;AAED;;GAEG;AACH,qBAAa,qBAAsB,YAAW,iBAAiB;IAI3D,OAAO,CAAC,QAAQ,CAAC,UAAU;IAE3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAN9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBAGR,UAAU,EAAE,WAAW,EACxC,SAAS,EAAE,MAAM,EACA,OAAO,EAAE,kBAAkB,EAC3B,WAAW,EAAE,WAAW;IAKrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC;IAuFzB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAkBxC,IAAI,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAIpC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAI1B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAI1B,YAAY,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoB9E"}
|
package/dist/storage/session.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { decryptPIIMap, encryptPIIMap } from "../crypto/index.js";
|
|
6
6
|
import { rehydrate as rehydrateText } from "../pipeline/tagger.js";
|
|
7
|
+
import { AnonymizerStream } from "../streaming/anonymizer-stream.js";
|
|
7
8
|
/**
|
|
8
9
|
* Session implementation that wraps an Anonymizer with automatic storage
|
|
9
10
|
*/
|
|
@@ -35,7 +36,8 @@ export class AnonymizerSessionImpl {
|
|
|
35
36
|
catch (error) {
|
|
36
37
|
// Decryption failed - likely key mismatch
|
|
37
38
|
const isKeyMismatch = error instanceof Error &&
|
|
38
|
-
(error.name === "OperationError" ||
|
|
39
|
+
(error.name === "OperationError" ||
|
|
40
|
+
error.message.includes("decrypt"));
|
|
39
41
|
if (isKeyMismatch) {
|
|
40
42
|
throw new Error(`Failed to decrypt existing session data for "${this.sessionId}". ` +
|
|
41
43
|
`The encryption key may have changed since this session was created.\n\n` +
|
|
@@ -54,6 +56,12 @@ export class AnonymizerSessionImpl {
|
|
|
54
56
|
}
|
|
55
57
|
// Call the parent anonymizer with existing PII map for ID reuse
|
|
56
58
|
const result = await this.anonymizer.anonymize(text, locale, policy, existingPiiMap);
|
|
59
|
+
// Sessions require a PII map (only available in pseudonymize mode)
|
|
60
|
+
if (result.piiMap === undefined) {
|
|
61
|
+
throw new Error("Session anonymize() failed: no PII map returned.\n\n" +
|
|
62
|
+
"This can happen if the anonymizer is in 'anonymize' mode.\n" +
|
|
63
|
+
"Sessions require 'pseudonymize' mode for PII map storage.");
|
|
64
|
+
}
|
|
57
65
|
// Decrypt the new PII map
|
|
58
66
|
const newPiiMap = await decryptPIIMap(result.piiMap, key);
|
|
59
67
|
// Merge maps: start with existing (if any), add new entries
|
|
@@ -100,5 +108,20 @@ export class AnonymizerSessionImpl {
|
|
|
100
108
|
async exists() {
|
|
101
109
|
return this.storage.exists(this.sessionId);
|
|
102
110
|
}
|
|
111
|
+
async createStream(config) {
|
|
112
|
+
// Load existing PII map to seed the stream for ID continuity
|
|
113
|
+
let initialPiiMap;
|
|
114
|
+
const existing = await this.storage.load(this.sessionId);
|
|
115
|
+
if (existing !== null) {
|
|
116
|
+
const key = await this.keyProvider.getKey();
|
|
117
|
+
initialPiiMap = await decryptPIIMap(existing.piiMap, key);
|
|
118
|
+
}
|
|
119
|
+
return new AnonymizerStream(this.anonymizer, {
|
|
120
|
+
...config,
|
|
121
|
+
sessionId: this.sessionId,
|
|
122
|
+
piiStorageProvider: this.storage,
|
|
123
|
+
keyProvider: this.keyProvider,
|
|
124
|
+
}, initialPiiMap);
|
|
125
|
+
}
|
|
103
126
|
}
|
|
104
127
|
//# sourceMappingURL=session.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/storage/session.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/storage/session.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAOnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAerE;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAIb;IAEA;IACA;IANV,SAAS,CAAS;IAE3B,YACmB,UAAuB,EACxC,SAAiB,EACA,OAA2B,EAC3B,WAAwB;QAHxB,eAAU,GAAV,UAAU,CAAa;QAEvB,YAAO,GAAP,OAAO,CAAoB;QAC3B,gBAAW,GAAX,WAAW,CAAa;QAEzC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAe,EACf,MAAqC;QAErC,yBAAyB;QACzB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAE5C,iEAAiE;QACjE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,cAAqC,CAAC;QAC1C,IAAI,SAAiB,CAAC;QACtB,IAAI,oBAA4C,CAAC;QAEjD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,cAAc,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC3D,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACxC,oBAAoB,GAAG,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;YACxD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,0CAA0C;gBAC1C,MAAM,aAAa,GACjB,KAAK,YAAY,KAAK;oBACtB,CAAC,KAAK,CAAC,IAAI,KAAK,gBAAgB;wBAC9B,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;gBAEvC,IAAI,aAAa,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,gDAAgD,IAAI,CAAC,SAAS,KAAK;wBACjE,yEAAyE;wBACzE,wBAAwB;wBACxB,6DAA6D;wBAC7D,uDAAuD;wBACvD,8DAA8D,CACjE,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,SAAS,CAAC;YAC3B,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,oBAAoB,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,gEAAgE;QAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAC5C,IAAI,EACJ,MAAM,EACN,MAAM,EACN,cAAc,CACf,CAAC;QAEF,mEAAmE;QACnE,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CACb,sDAAsD;gBACpD,6DAA6D;gBAC7D,2DAA2D,CAC9D,CAAC;QACJ,CAAC;QAED,0BAA0B;QAC1B,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAE1D,4DAA4D;QAC5D,MAAM,YAAY,GAAc,cAAc;YAC5C,CAAC,CAAC,IAAI,GAAG,CAAiB,cAAc,CAAC;YACzC,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAC;QAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;YAC/B,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,sBAAsB;QACtB,MAAM,kBAAkB,GAAG,EAAE,GAAG,oBAAoB,EAAE,CAAC;QACvD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;YACtE,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;QACrE,CAAC;QAED,gCAAgC;QAChC,MAAM,kBAAkB,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QAElE,qCAAqC;QACrC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,EAAE;YAC1D,SAAS;YACT,YAAY,EAAE,kBAAkB;YAChC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;SACxC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC1B,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,iCAAiC,IAAI,CAAC,SAAS,IAAI;gBACjD,mDAAmD,CACtD,CAAC;QACJ,CAAC;QAED,sBAAsB;QACtB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAEvD,qBAAqB;QACrB,OAAO,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAA8B;QAC/C,6DAA6D;QAC7D,IAAI,aAAoC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAC5C,aAAa,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,gBAAgB,CACzB,IAAI,CAAC,UAAU,EACf;YACE,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,kBAAkB,EAAE,IAAI,CAAC,OAAO;YAChC,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,EACD,aAAa,CACd,CAAC;IACJ,CAAC;CACF"}
|
package/dist/storage/types.d.ts
CHANGED
|
@@ -151,5 +151,21 @@ export interface AnonymizerSession {
|
|
|
151
151
|
* @returns true if exists, false otherwise
|
|
152
152
|
*/
|
|
153
153
|
exists(): Promise<boolean>;
|
|
154
|
+
/**
|
|
155
|
+
* Create a streaming anonymizer bound to this session.
|
|
156
|
+
* The stream automatically uses this session's ID, storage, and key provider.
|
|
157
|
+
* Node.js/Bun only — not available in browser builds.
|
|
158
|
+
*
|
|
159
|
+
* @param config - Optional stream configuration overrides
|
|
160
|
+
* @returns A Transform stream that anonymizes text chunk-by-chunk
|
|
161
|
+
*
|
|
162
|
+
* @example
|
|
163
|
+
* ```typescript
|
|
164
|
+
* const session = anonymizer.session('chat-123');
|
|
165
|
+
* const stream = await session.createStream();
|
|
166
|
+
* input.pipe(stream).pipe(output);
|
|
167
|
+
* ```
|
|
168
|
+
*/
|
|
169
|
+
createStream?(config?: Record<string, unknown>): Promise<NodeJS.ReadWriteStream>;
|
|
154
170
|
}
|
|
155
171
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EACV,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAE3B;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,8CAA8C;IAC9C,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,iCAAiC;IACjC,MAAM,EAAE,eAAe,CAAC;IACxB,0BAA0B;IAC1B,QAAQ,EAAE,cAAc,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mDAAmD;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,SAAS,CAAC,EAAE,IAAI,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;OAMG;IACH,IAAI,CACF,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;OAIG;IACH,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAE3D;;;;OAIG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjD;;;;OAIG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjD;;;;OAIG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/C;;;;;OAKG;IACH,OAAO,CAAC,SAAS,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,iBAAiB;IAChC,6BAA6B;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;;;;;OAMG;IACH,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAEhC;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEzC;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAErC;;;OAGG;IACH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3B;;;OAGG;IACH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EACV,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAE3B;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,8CAA8C;IAC9C,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,iCAAiC;IACjC,MAAM,EAAE,eAAe,CAAC;IACxB,0BAA0B;IAC1B,QAAQ,EAAE,cAAc,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mDAAmD;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,SAAS,CAAC,EAAE,IAAI,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;OAMG;IACH,IAAI,CACF,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;OAIG;IACH,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAE3D;;;;OAIG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjD;;;;OAIG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjD;;;;OAIG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/C;;;;;OAKG;IACH,OAAO,CAAC,SAAS,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,iBAAiB;IAChC,6BAA6B;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;;;;;OAMG;IACH,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAEhC;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEzC;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAErC;;;OAGG;IACH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3B;;;OAGG;IACH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3B;;;;;;;;;;;;;;OAcG;IACH,YAAY,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;CAClF"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anonymizer Stream
|
|
3
|
+
* Node.js Transform stream that anonymizes text chunk-by-chunk
|
|
4
|
+
* using sentence buffering for NER context.
|
|
5
|
+
*/
|
|
6
|
+
import { Transform, type TransformCallback } from "node:stream";
|
|
7
|
+
import type { RawPIIMap } from "../pipeline/tagger.js";
|
|
8
|
+
import type { StreamConfig } from "./types.js";
|
|
9
|
+
import type { IAnonymizer } from "../storage/session.js";
|
|
10
|
+
/**
|
|
11
|
+
* Transform stream that anonymizes text passing through it.
|
|
12
|
+
*
|
|
13
|
+
* Uses a SentenceBuffer to accumulate text and flush at sentence
|
|
14
|
+
* boundaries, maintaining overlap for NER context.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```typescript
|
|
18
|
+
* const stream = await createAnonymizerStream({
|
|
19
|
+
* anonymizer: { ner: { mode: 'quantized' } },
|
|
20
|
+
* sessionId: 'chat-123',
|
|
21
|
+
* piiStorageProvider: storage,
|
|
22
|
+
* keyProvider: keyProvider,
|
|
23
|
+
* });
|
|
24
|
+
*
|
|
25
|
+
* createReadStream('input.txt')
|
|
26
|
+
* .pipe(stream)
|
|
27
|
+
* .pipe(createWriteStream('anonymized.txt'));
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
30
|
+
export declare class AnonymizerStream extends Transform {
|
|
31
|
+
private readonly sentenceBuffer;
|
|
32
|
+
private readonly keyProvider;
|
|
33
|
+
private readonly piiStorageProvider;
|
|
34
|
+
private readonly sessionId;
|
|
35
|
+
private readonly onChunkCallback?;
|
|
36
|
+
private readonly onFinishCallback?;
|
|
37
|
+
private readonly saveIntervalMs;
|
|
38
|
+
private lastSaveTime;
|
|
39
|
+
private totalEntities;
|
|
40
|
+
private startTime;
|
|
41
|
+
private chunkCount;
|
|
42
|
+
constructor(anonymizer: IAnonymizer, config: StreamConfig, initialPiiMap?: RawPIIMap);
|
|
43
|
+
/**
|
|
44
|
+
* Get the cumulative PII map built across all chunks.
|
|
45
|
+
* Available after stream finishes.
|
|
46
|
+
*/
|
|
47
|
+
getPiiMap(): RawPIIMap;
|
|
48
|
+
/**
|
|
49
|
+
* Get stream statistics.
|
|
50
|
+
*/
|
|
51
|
+
get stats(): {
|
|
52
|
+
totalEntities: number;
|
|
53
|
+
chunksProcessed: number;
|
|
54
|
+
totalProcessingTimeMs: number;
|
|
55
|
+
};
|
|
56
|
+
_transform(chunk: Buffer | string, encoding: BufferEncoding, callback: TransformCallback): void;
|
|
57
|
+
_flush(callback: TransformCallback): void;
|
|
58
|
+
private processChunk;
|
|
59
|
+
private maybeSaveToStorage;
|
|
60
|
+
private processFlush;
|
|
61
|
+
private buildEntityCounts;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=anonymizer-stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anonymizer-stream.d.ts","sourceRoot":"","sources":["../../src/streaming/anonymizer-stream.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,KAAK,EAAE,YAAY,EAAuC,MAAM,YAAY,CAAC;AACpF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAKzD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,gBAAiB,SAAQ,SAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA4B;IAC/D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAoC;IACrE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAqC;IAEvE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAC/C,OAAO,CAAC,YAAY,CAAK;IAEzB,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,UAAU,CAAK;gBAGrB,UAAU,EAAE,WAAW,EACvB,MAAM,EAAE,YAAY,EACpB,aAAa,CAAC,EAAE,SAAS;IAuB3B;;;OAGG;IACH,SAAS,IAAI,SAAS;IAItB;;OAEG;IACH,IAAI,KAAK,IAAI;QACX,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAOA;IAEQ,UAAU,CACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,QAAQ,EAAE,cAAc,EACxB,QAAQ,EAAE,iBAAiB,GAC1B,IAAI;IAYE,MAAM,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI;YAMpC,YAAY;YAuBZ,kBAAkB;YAsBlB,YAAY;IAgD1B,OAAO,CAAC,iBAAiB;CAY1B"}
|